Import libraries

library(tidyverse)
library(tidymodels)
library(solitude) # -- new package 
library(janitor)
library(ggpubr)
library(skimr)
library(themis)
library(dplyr)
library(vip)
library(DALEX)    # new 
library(DALEXtra) # new
library(rpart)
library(rpart.plot)
loan <- read_csv("loan_train.csv") %>%
  clean_names()
Rows: 29777 Columns: 52── Column specification ──────────────────────────────────────────────────────────────────────────────────────────────────────────
Delimiter: ","
chr (23): term, int_rate, grade, sub_grade, emp_title, emp_length, home_ownership, verification_status, issue_d, loan_status, ...
dbl (29): id, member_id, loan_amnt, funded_amnt, funded_amnt_inv, installment, annual_inc, dti, delinq_2yrs, fico_range_low, f...
ℹ Use `spec()` to retrieve the full column specification for this data.
ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
head(loan)
skim(loan)
── Data Summary ────────────────────────
                           Values
Name                       loan  
Number of rows             29777 
Number of columns          52    
_______________________          
Column type frequency:           
  character                23    
  numeric                  29    
________________________         
Group variables            None  
kaggle <- read_csv("loan_holdout.csv") %>% clean_names()
Rows: 12761 Columns: 51── Column specification ──────────────────────────────────────────────────────────────────────────────────────────────────────────
Delimiter: ","
chr (22): term, int_rate, grade, sub_grade, emp_title, emp_length, home_ownership, verification_status, issue_d, pymnt_plan, u...
dbl (29): id, member_id, loan_amnt, funded_amnt, funded_amnt_inv, installment, annual_inc, dti, delinq_2yrs, fico_range_low, f...
ℹ Use `spec()` to retrieve the full column specification for this data.
ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
skim(kaggle)
── Data Summary ────────────────────────
                           Values
Name                       kaggle
Number of rows             12761 
Number of columns          51    
_______________________          
Column type frequency:           
  character                22    
  numeric                  29    
________________________         
Group variables            None  

explore

n_cols <- names(loan %>% select_if(is.numeric) %>% select(-id,-member_id))

my_hist <- function(col){
  loan %>%
    summarise(n=n(), 
              n_miss = sum(is.na(!!as.name(col))),
              n_dist = n_distinct(!!as.name(col)),
              mean = round(mean(!!as.name(col), na.rm=TRUE),2),
              min  = min(!!as.name(col), na.rm=TRUE),
              max  = max(!!as.name(col), na.rm=TRUE)
              ) -> col_summary
  
   p1  <- ggtexttable(col_summary, rows = NULL, 
                        theme = ttheme("mOrange"))
  
h1 <- loan %>%
  ggplot(aes(x=!!as.name(col))) +
  geom_histogram(bins=30) 

plt <- ggarrange( h1, p1, 
          ncol = 1, nrow = 2,
          heights = c(1, 0.3)) 

print(plt)

}

for (c in n_cols){
  my_hist(c)
}
Warning: Removed 3 rows containing non-finite values (stat_bin).

explore target

loan_summary <- loan %>%
  count(loan_status) %>%
  mutate(pct = n/sum(n))


loan_summary %>%
  ggplot(aes(x=factor(loan_status),y=pct)) +
  geom_col()  + 
  geom_text(aes(label = round(pct*100,2)) , vjust = 2.5, colour = "white") + 
  labs(title="Loan Status Plot", x="Loan_Status", y="PCT")

NA
NA

Explotary Analysis

loan_vis <- loan %>% 
   mutate_if(is.character, factor) 

for (c in names(loan_vis %>% dplyr::select(!c(id,member_id)))) {
  if (c == "event_timestamp") {
   # print( fraud_vis %>%
             #ggplot(., aes(!!as.name(c))) + 
             #geom_histogram(aes(bins=10,fill = loan_status), position = "fill")  +labs(title = c, y = "pct fraud"))
      
  }else if (c %in% names(loan_vis %>% dplyr::select(where(is.factor)))) {
    # -- for each character column create a chart
    print( loan_vis %>%
             ggplot(., aes(!!as.name(c))) + 
             geom_bar(aes(fill = loan_status), position = "fill")  + labs(title = c, y = "pct fraud"))
  } else {
    # -- comparative boxplots
    print(ggplot(loan_vis, aes(x=loan_status, y=!!as.name(c), fill=loan_status))+ geom_boxplot() +labs(title = c))
  }
}
[WARNING] Deprecated: --self-contained. use --embed-resources --standalone
[WARNING] Deprecated: --self-contained. use --embed-resources --standalone
[WARNING] Deprecated: --self-contained. use --embed-resources --standalone

#correlation

library(reshape2)
loan_numeric <- subset(loan,select = -c(id,member_id)) %>%
  select_if(.,is.numeric)

cor_mat <- loan_numeric %>%
  cor()


cor_melt <- cor_mat %>% melt 

cor_melt %>%
  mutate(value = round(value,2)) %>%
 ggplot(aes(Var2, Var1, fill = value))+
 geom_tile() +
 scale_fill_gradient2(low = "blue", high = "red", mid = "white", 
                      midpoint = 0, limit = c(-1,1), space = "Lab", 
                      name="Correlation") +
 theme_minimal() +
 theme(axis.text.x = element_text(angle = 45, vjust = 1, 
                                  size = 4, hjust = 1),axis.text.y = element_text(angle = 45, vjust = 1, 
                                  size = 4, hjust = 1))+
 coord_fixed() +
 geom_text(aes(Var2, Var1, label = value), color = "black", size = 1.5) +
  labs(title = "Pearson Correlation for Numerical Data")

Recipe

# deal w. categoricals 
loan_recipe <- recipe(~.,loan) %>%
  step_rm(id,member_id,int_rate,emp_title,url,desc,title,zip_code,earliest_cr_line,revol_util,mths_since_last_delinq,mths_since_last_record,next_pymnt_d) %>%
  step_unknown(all_nominal_predictors()) %>%
  step_impute_median(all_numeric_predictors()) %>%
  step_dummy(all_nominal_predictors()) %>%
  prep()

bake_loan <- bake(loan_recipe, loan)

Train your IsolationForest

iso_forest <- isolationForest$new(
  sample_size = 1000,
  num_trees = 100,
  max_depth = ceiling(log2(1000)))


iso_forest$fit(bake_loan)
INFO  [17:34:53.812] dataset has duplicated rows
INFO  [17:34:53.817] Building Isolation Forest ...
INFO  [17:34:56.876] done
INFO  [17:34:56.879] Computing depth of terminal nodes ...
INFO  [17:34:57.094] done
INFO  [17:34:58.301] Completed growing isolation forest

predict training

evaluate histogram pick a value of average_depth to identify anomalies. a shorter average depth means the point is more isolated and more likely an anomaly

pred_train <- iso_forest$predict(bake_loan)

pred_train %>%
  ggplot(aes(average_depth)) +
  geom_histogram(bins=20) + 
  geom_vline(xintercept = 9.45, linetype="dotted", 
                color = "blue", size=1.5) + 
  labs(title="Isolation Forest Average Tree Depth")


pred_train %>%
  ggplot(aes(anomaly_score)) +
  geom_histogram(bins=20) + 
  geom_vline(xintercept = 0.6, linetype="dotted", 
                color = "blue", size=1.5) + 
  labs(title="Isolation Forest Anomaly Score Above 0.6")

NA
NA

global level interpretation

The steps of interpreting anomalies on a global level are:

  1. Create a data frame with a column that indicates whether the record was considered an anomaly.
  2. Train a decision tree to predict the anomaly flag.
  3. Visualize the decision tree to determine which segments of the data are considered anomalous.
train_pred <- bind_cols(iso_forest$predict(bake_loan),bake_loan) %>%
  mutate(anomaly = as.factor(if_else(average_depth <= 9.45, "Anomaly","Normal")))

train_pred %>%
  arrange(average_depth) %>%
  count(anomaly)
NA

Fit a Tree

fmla <- as.formula(paste("anomaly ~ ", paste(bake_loan %>% colnames(), collapse= "+")))

outlier_tree <- decision_tree(min_n=2, tree_depth=3, cost_complexity = .01) %>%
  set_mode("classification") %>%
  set_engine("rpart") %>%
  fit(fmla, data=train_pred)

outlier_tree$fit
n= 29777 

node), split, n, loss, yval, (yprob)
      * denotes terminal node

 1) root 29777 6 Normal (0.0002014978 0.9997985022)  
   2) last_pymnt_amnt>=34166.69 14 1 Normal (0.0714285714 0.9285714286)  
     4) open_acc>=24 1 0 Anomaly (1.0000000000 0.0000000000) *
     5) open_acc< 24 13 0 Normal (0.0000000000 1.0000000000) *
   3) last_pymnt_amnt< 34166.69 29763 5 Normal (0.0001679938 0.9998320062)  
     6) revol_bal>=214867 45 1 Normal (0.0222222222 0.9777777778)  
      12) revol_bal< 215273.5 1 0 Anomaly (1.0000000000 0.0000000000) *
      13) revol_bal>=215273.5 44 0 Normal (0.0000000000 1.0000000000) *
     7) revol_bal< 214867 29718 4 Normal (0.0001345986 0.9998654014) *
library(rpart.plot) # -- plotting decision trees 

rpart.plot(outlier_tree$fit,clip.right.labs = FALSE, branch = .3, under = TRUE, roundint=FALSE, extra=3)

Global Anomaly Rules

anomaly_rules <- rpart.rules(outlier_tree$fit,roundint=FALSE, extra = 4, cover = TRUE, clip.facs = TRUE) %>% clean_names() %>%
  #filter(anomaly=="Anomaly") %>%
  mutate(rule = "IF") 


rule_cols <- anomaly_rules %>% select(starts_with("x_")) %>% colnames()

for (col in rule_cols){
anomaly_rules <- anomaly_rules %>%
    mutate(rule = paste(rule, !!as.name(col)))
}

anomaly_rules %>%
  as.data.frame() %>%
  filter(anomaly == "Anomaly") %>%
  mutate(rule = paste(rule, " THEN ", anomaly )) %>%
  mutate(rule = paste(rule," coverage ", cover)) %>%
  select( rule)

anomaly_rules %>%
  as.data.frame() %>%
  filter(anomaly == "Normal") %>%
  mutate(rule = paste(rule, " THEN ", anomaly )) %>%
  mutate(rule = paste(rule," coverage ", cover)) %>%
  select( rule)

pred_train <- bind_cols(iso_forest$predict(bake_loan),
                        bake_loan)


pred_train %>%
  arrange(desc(anomaly_score) ) %>%
  filter(average_depth <= 9.45)

Local Anomaly Rules


fmla <- as.formula(paste("anomaly ~ ", paste(bake_loan %>% colnames(), collapse= "+")))

pred_train %>%
  mutate(anomaly= as.factor(if_else(id==28576, "Anomaly", "Normal"))) -> local_df

local_tree <-  decision_tree(mode="classification",
                            tree_depth = 5,
                            min_n = 1,
                            cost_complexity=0) %>%
              set_engine("rpart") %>%
                  fit(fmla,local_df )

local_tree$fit
n= 29777 

node), split, n, loss, yval, (yprob)
      * denotes terminal node

1) root 29777 1 Normal (3.358297e-05 9.999664e-01)  
  2) addr_state_SD>=0.5 48 1 Normal (2.083333e-02 9.791667e-01)  
    4) annual_inc>=102052 1 0 Anomaly (1.000000e+00 0.000000e+00) *
    5) annual_inc< 102052 47 0 Normal (0.000000e+00 1.000000e+00) *
  3) addr_state_SD< 0.5 29729 0 Normal (0.000000e+00 1.000000e+00) *
rpart.rules(local_tree$fit, extra = 4, cover = TRUE, clip.facs = TRUE, roundint=FALSE)
rpart.plot(local_tree$fit, roundint=FALSE, extra=3)


anomaly_rules <- rpart.rules(local_tree$fit, extra = 4, cover = TRUE, clip.facs = TRUE) %>% clean_names() %>%
  filter(anomaly=="Anomaly") %>%
  mutate(rule = "IF") 
Warning: Cannot retrieve the data used to build the model (so cannot determine roundint and is.binary for the variables).
To silence this warning:
    Call rpart.rules with roundint=FALSE,
    or rebuild the rpart model with model=TRUE.
rule_cols <- anomaly_rules %>% select(starts_with("x_")) %>% colnames()

for (col in rule_cols){
anomaly_rules <- anomaly_rules %>%
    mutate(rule = paste(rule, !!as.name(col)))
}

as.data.frame(anomaly_rules) %>%
  select(rule, cover)

local_df %>%
  filter(addr_state_SD >=0.5) %>%
  filter(annual_inc >=102052) %>%
  summarise(n=n(),
            mean_annual_inc = mean(annual_inc))
local_explainer <- function(ID){
  
  fmla <- as.formula(paste("anomaly ~ ", paste(bake_loan %>% colnames(), collapse= "+")))
  
  pred_train %>%
    mutate(anomaly= as.factor(if_else(id==ID, "Anomaly", "Normal"))) -> local_df
  
  local_tree <-  decision_tree(mode="classification",
                              tree_depth = 3,
                              min_n = 1,
                              cost_complexity=0) %>%
                set_engine("rpart") %>%
                    fit(fmla,local_df )
  
  local_tree$fit
  
  #rpart.rules(local_tree$fit, extra = 4, cover = TRUE, clip.facs = TRUE)
  rpart.plot(local_tree$fit, roundint=FALSE, extra=3) %>% print()
  
  anomaly_rules <- rpart.rules(local_tree$fit, extra = 4, cover = TRUE, clip.facs = TRUE) %>% clean_names() %>%
    filter(anomaly=="Anomaly") %>%
    mutate(rule = "IF") 
  
  
  rule_cols <- anomaly_rules %>% select(starts_with("x_")) %>% colnames()
  
  for (col in rule_cols){
  anomaly_rules <- anomaly_rules %>%
      mutate(rule = paste(rule, !!as.name(col)))
  }
  
  as.data.frame(anomaly_rules) %>%
    select(rule, cover) %>%
    print()
}

pred_train %>%
  filter(average_depth <=9.45) %>%
  pull(id) -> anomaly_vect

for (anomaly_id in anomaly_vect){
  #print(anomaly_id)
  local_explainer(anomaly_id)
}
$obj
n= 29777 

node), split, n, loss, yval, (yprob)
      * denotes terminal node

1) root 29777 1 Normal (3.358297e-05 9.999664e-01)  
  2) last_pymnt_amnt>=34166.69 14 1 Normal (7.142857e-02 9.285714e-01)  
    4) open_acc>=24 1 0 Anomaly (1.000000e+00 0.000000e+00) *
    5) open_acc< 24 13 0 Normal (0.000000e+00 1.000000e+00) *
  3) last_pymnt_amnt< 34166.69 29763 0 Normal (0.000000e+00 1.000000e+00) *

$snipped.nodes
NULL

$xlim
[1] 0 1

$ylim
[1] 0 1

$x
[1] 0.60786912 0.27647172 0.05554013 0.49740332 0.93926651

$y
[1] 0.92241378 0.52076802 0.03879311 0.03879311 0.03879311

$branch.x
       [,1]      [,2]       [,3]      [,4]      [,5]
x 0.6078691 0.2764717 0.05554013 0.4974033 0.9392665
         NA 0.2764717 0.05554013 0.4974033 0.9392665
         NA 0.6078691 0.27647172 0.2764717 0.6078691

$branch.y
      [,1]      [,2]      [,3]      [,4]      [,5]
y 1.001438 0.5997924 0.1178175 0.1178175 0.1178175
        NA 0.8310967 0.4294509 0.4294509 0.8310967
        NA 0.8310967 0.4294509 0.4294509 0.8310967

$labs
[1] "Normal\n1 / 29777" "Normal\n1 / 14"    "Anomaly\n0 / 1"    "Normal\n0 / 13"    "Normal\n0 / 29763"

$cex
[1] 1

$boxes
$boxes$x1
[1]  0.541919895  0.224887678 -0.004532432  0.445819274  0.873317289

$boxes$y1
[1]  0.878511331  0.476865573 -0.005109338 -0.005109338 -0.005109338

$boxes$x2
[1] 0.6738183 0.3280558 0.1156127 0.5489874 1.0052157

$boxes$y2
[1] 1.0014382 0.5997924 0.1178175 0.1178175 0.1178175


$split.labs
[1] ""

$split.cex
[1] 1 1 1 1 1

$split.box
$split.box$x1
[1] 0.4309163 0.1680799        NA        NA        NA

$split.box$y1
[1] 0.7959747 0.3943290        NA        NA        NA

$split.box$x2
[1] 0.7848220 0.3848635        NA        NA        NA

$split.box$y2
[1] 0.8662186 0.4645729        NA        NA        NA

$obj
n= 29777 

node), split, n, loss, yval, (yprob)
      * denotes terminal node

1) root 29777 1 Normal (3.358297e-05 9.999664e-01)  
  2) sub_grade_G1>=0.5 102 1 Normal (9.803922e-03 9.901961e-01)  
    4) purpose_medical>=0.5 1 0 Anomaly (1.000000e+00 0.000000e+00) *
    5) purpose_medical< 0.5 101 0 Normal (0.000000e+00 1.000000e+00) *
  3) sub_grade_G1< 0.5 29675 0 Normal (0.000000e+00 1.000000e+00) *

$snipped.nodes
NULL

$xlim
[1] 0 1

$ylim
[1] 0 1

$x
[1] 0.60786912 0.27647172 0.05554013 0.49740332 0.93926651

$y
[1] 0.92241378 0.52076802 0.03879311 0.03879311 0.03879311

$branch.x
       [,1]      [,2]       [,3]      [,4]      [,5]
x 0.6078691 0.2764717 0.05554013 0.4974033 0.9392665
         NA 0.2764717 0.05554013 0.4974033 0.9392665
         NA 0.6078691 0.27647172 0.2764717 0.6078691

$branch.y
      [,1]      [,2]      [,3]      [,4]      [,5]
y 1.001438 0.5997924 0.1178175 0.1178175 0.1178175
        NA 0.8310967 0.4294509 0.4294509 0.8310967
        NA 0.8310967 0.4294509 0.4294509 0.8310967

$labs
[1] "Normal\n1 / 29777" "Normal\n1 / 102"   "Anomaly\n0 / 1"    "Normal\n0 / 101"   "Normal\n0 / 29675"

$cex
[1] 1

$boxes
$boxes$x1
[1]  0.541919895  0.224887678 -0.004532432  0.445819274  0.873317289

$boxes$y1
[1]  0.878511331  0.476865573 -0.005109338 -0.005109338 -0.005109338

$boxes$x2
[1] 0.6738183 0.3280558 0.1156127 0.5489874 1.0052157

$boxes$y2
[1] 1.0014382 0.5997924 0.1178175 0.1178175 0.1178175


$split.labs
[1] ""

$split.cex
[1] 1 1 1 1 1

$split.box
$split.box$x1
[1] 0.4648703 0.1184548        NA        NA        NA

$split.box$y1
[1] 0.7959747 0.3943290        NA        NA        NA

$split.box$x2
[1] 0.7508679 0.4344887        NA        NA        NA

$split.box$y2
[1] 0.8662186 0.4645729        NA        NA        NA

$obj
n= 29777 

node), split, n, loss, yval, (yprob)
      * denotes terminal node

1) root 29777 1 Normal (3.358297e-05 9.999664e-01)  
  2) last_credit_pull_d_Jun.13>=0.5 224 1 Normal (4.464286e-03 9.955357e-01)  
    4) sub_grade_C2>=0.5 8 1 Normal (1.250000e-01 8.750000e-01)  
      8) fico_range_low>=705 1 0 Anomaly (1.000000e+00 0.000000e+00) *
      9) fico_range_low< 705 7 0 Normal (0.000000e+00 1.000000e+00) *
    5) sub_grade_C2< 0.5 216 0 Normal (0.000000e+00 1.000000e+00) *
  3) last_credit_pull_d_Jun.13< 0.5 29553 0 Normal (0.000000e+00 1.000000e+00) *

$snipped.nodes
NULL

$xlim
[1] 0 1

$ylim
[1] 0 1

$x
[1] 0.68151298 0.42375945 0.20282786 0.05554013 0.35011559 0.64469105 0.93926651

$y
[1] 0.92241378 0.65464994 0.38688610 0.03879311 0.03879311 0.03879311 0.03879311

$branch.x
      [,1]      [,2]      [,3]       [,4]      [,5]      [,6]      [,7]
x 0.681513 0.4237595 0.2028279 0.05554013 0.3501156 0.6446910 0.9392665
        NA 0.4237595 0.2028279 0.05554013 0.3501156 0.6446910 0.9392665
        NA 0.6815130 0.4237595 0.20282786 0.2028279 0.4237595 0.6815130

$branch.y
      [,1]      [,2]      [,3]      [,4]      [,5]      [,6]      [,7]
y 1.001438 0.7336743 0.4659105 0.1178175 0.1178175 0.1178175 0.1178175
        NA 0.8310967 0.5633328 0.2955690 0.2955690 0.5633328 0.8310967
        NA 0.8310967 0.5633328 0.2955690 0.2955690 0.5633328 0.8310967

$labs
[1] "Normal\n1 / 29777" "Normal\n1 / 224"   "Normal\n1 / 8"     "Anomaly\n0 / 1"    "Normal\n0 / 7"     "Normal\n0 / 216"  
[7] "Normal\n0 / 29553"

$cex
[1] 1

$boxes
$boxes$x1
[1]  0.615563760  0.372175408  0.151243812 -0.004532432  0.298531543  0.593107005  0.873317289

$boxes$y1
[1]  0.878511331  0.610747492  0.342983653 -0.005109338 -0.005109338 -0.005109338 -0.005109338

$boxes$x2
[1] 0.7474622 0.4753435 0.2544119 0.1156127 0.4016996 0.6962751 1.0052157

$boxes$y2
[1] 1.0014382 0.7336743 0.4659105 0.1178175 0.1178175 0.1178175 0.1178175


$split.labs
[1] ""

$split.cex
[1] 1 1 1 1 1 1 1

$split.box
$split.box$x1
[1] 0.47125903 0.28206657 0.05329942         NA         NA         NA         NA

$split.box$y1
[1] 0.7959747 0.5282109 0.2604471        NA        NA        NA        NA

$split.box$x2
[1] 0.8917669 0.5654523 0.3523563        NA        NA        NA        NA

$split.box$y2
[1] 0.8662186 0.5984548 0.3306910        NA        NA        NA        NA

$obj
n= 29777 

node), split, n, loss, yval, (yprob)
      * denotes terminal node

1) root 29777 1 Normal (3.358297e-05 9.999664e-01)  
  2) last_pymnt_d_Dec.10>=0.5 233 1 Normal (4.291845e-03 9.957082e-01)  
    4) funded_amnt_inv>=23591 2 1 Anomaly (5.000000e-01 5.000000e-01)  
      8) loan_amnt< 24125 1 0 Anomaly (1.000000e+00 0.000000e+00) *
      9) loan_amnt>=24125 1 0 Normal (0.000000e+00 1.000000e+00) *
    5) funded_amnt_inv< 23591 231 0 Normal (0.000000e+00 1.000000e+00) *
  3) last_pymnt_d_Dec.10< 0.5 29544 0 Normal (0.000000e+00 1.000000e+00) *

$snipped.nodes
NULL

$xlim
[1] 0 1

$ylim
[1] 0 1

$x
[1] 0.68151298 0.42375945 0.20282786 0.05554013 0.35011559 0.64469105 0.93926651

$y
[1] 0.92241378 0.65464994 0.38688610 0.03879311 0.03879311 0.03879311 0.03879311

$branch.x
      [,1]      [,2]      [,3]       [,4]      [,5]      [,6]      [,7]
x 0.681513 0.4237595 0.2028279 0.05554013 0.3501156 0.6446910 0.9392665
        NA 0.4237595 0.2028279 0.05554013 0.3501156 0.6446910 0.9392665
        NA 0.6815130 0.4237595 0.20282786 0.2028279 0.4237595 0.6815130

$branch.y
      [,1]      [,2]      [,3]      [,4]      [,5]      [,6]      [,7]
y 1.001438 0.7336743 0.4659105 0.1178175 0.1178175 0.1178175 0.1178175
        NA 0.8310967 0.5633328 0.2955690 0.2955690 0.5633328 0.8310967
        NA 0.8310967 0.5633328 0.2955690 0.2955690 0.5633328 0.8310967

$labs
[1] "Normal\n1 / 29777" "Normal\n1 / 233"   "Anomaly\n1 / 2"    "Anomaly\n0 / 1"    "Normal\n0 / 1"     "Normal\n0 / 231"  
[7] "Normal\n0 / 29544"

$cex
[1] 1

$boxes
$boxes$x1
[1]  0.615563760  0.372175408  0.142755298 -0.004532432  0.298531543  0.593107005  0.873317289

$boxes$y1
[1]  0.878511331  0.610747492  0.342983653 -0.005109338 -0.005109338 -0.005109338 -0.005109338

$boxes$x2
[1] 0.7474622 0.4753435 0.2629004 0.1156127 0.4016996 0.6962751 1.0052157

$boxes$y2
[1] 1.0014382 0.7336743 0.4659105 0.1178175 0.1178175 0.1178175 0.1178175


$split.labs
[1] ""

$split.cex
[1] 1 1 1 1 1 1 1

$split.box
$split.box$x1
[1] 0.49672457 0.24615363 0.07550015         NA         NA         NA         NA

$split.box$y1
[1] 0.7959747 0.5282109 0.2604471        NA        NA        NA        NA

$split.box$x2
[1] 0.8663014 0.6013653 0.3301556        NA        NA        NA        NA

$split.box$y2
[1] 0.8662186 0.5984548 0.3306910        NA        NA        NA        NA

$obj
n= 29777 

node), split, n, loss, yval, (yprob)
      * denotes terminal node

1) root 29777 1 Normal (3.358297e-05 9.999664e-01)  
  2) revol_bal>=214867 45 1 Normal (2.222222e-02 9.777778e-01)  
    4) revol_bal< 215273.5 1 0 Anomaly (1.000000e+00 0.000000e+00) *
    5) revol_bal>=215273.5 44 0 Normal (0.000000e+00 1.000000e+00) *
  3) revol_bal< 214867 29732 0 Normal (0.000000e+00 1.000000e+00) *

$snipped.nodes
NULL

$xlim
[1] 0 1

$ylim
[1] 0 1

$x
[1] 0.60786912 0.27647172 0.05554013 0.49740332 0.93926651

$y
[1] 0.92241378 0.52076802 0.03879311 0.03879311 0.03879311

$branch.x
       [,1]      [,2]       [,3]      [,4]      [,5]
x 0.6078691 0.2764717 0.05554013 0.4974033 0.9392665
         NA 0.2764717 0.05554013 0.4974033 0.9392665
         NA 0.6078691 0.27647172 0.2764717 0.6078691

$branch.y
      [,1]      [,2]      [,3]      [,4]      [,5]
y 1.001438 0.5997924 0.1178175 0.1178175 0.1178175
        NA 0.8310967 0.4294509 0.4294509 0.8310967
        NA 0.8310967 0.4294509 0.4294509 0.8310967

$labs
[1] "Normal\n1 / 29777" "Normal\n1 / 45"    "Anomaly\n0 / 1"    "Normal\n0 / 44"    "Normal\n0 / 29732"

$cex
[1] 1

$boxes
$boxes$x1
[1]  0.541919895  0.224887678 -0.004532432  0.445819274  0.873317289

$boxes$y1
[1]  0.878511331  0.476865573 -0.005109338 -0.005109338 -0.005109338

$boxes$x2
[1] 0.6738183 0.3280558 0.1156127 0.5489874 1.0052157

$boxes$y2
[1] 1.0014382 0.5997924 0.1178175 0.1178175 0.1178175


$split.labs
[1] ""

$split.cex
[1] 1 1 1 1 1

$split.box
$split.box$x1
[1] 0.4733588 0.1497970        NA        NA        NA

$split.box$y1
[1] 0.7959747 0.3943290        NA        NA        NA

$split.box$x2
[1] 0.7423794 0.4031465        NA        NA        NA

$split.box$y2
[1] 0.8662186 0.4645729        NA        NA        NA

$obj
n= 29777 

node), split, n, loss, yval, (yprob)
      * denotes terminal node

1) root 29777 1 Normal (3.358297e-05 9.999664e-01)  
  2) addr_state_SD>=0.5 48 1 Normal (2.083333e-02 9.791667e-01)  
    4) annual_inc>=102052 1 0 Anomaly (1.000000e+00 0.000000e+00) *
    5) annual_inc< 102052 47 0 Normal (0.000000e+00 1.000000e+00) *
  3) addr_state_SD< 0.5 29729 0 Normal (0.000000e+00 1.000000e+00) *

$snipped.nodes
NULL

$xlim
[1] 0 1

$ylim
[1] 0 1

$x
[1] 0.60786912 0.27647172 0.05554013 0.49740332 0.93926651

$y
[1] 0.92241378 0.52076802 0.03879311 0.03879311 0.03879311

$branch.x
       [,1]      [,2]       [,3]      [,4]      [,5]
x 0.6078691 0.2764717 0.05554013 0.4974033 0.9392665
         NA 0.2764717 0.05554013 0.4974033 0.9392665
         NA 0.6078691 0.27647172 0.2764717 0.6078691

$branch.y
      [,1]      [,2]      [,3]      [,4]      [,5]
y 1.001438 0.5997924 0.1178175 0.1178175 0.1178175
        NA 0.8310967 0.4294509 0.4294509 0.8310967
        NA 0.8310967 0.4294509 0.4294509 0.8310967

$labs
[1] "Normal\n1 / 29777" "Normal\n1 / 48"    "Anomaly\n0 / 1"    "Normal\n0 / 47"    "Normal\n0 / 29729"

$cex
[1] 1

$boxes
$boxes$x1
[1]  0.541919895  0.224887678 -0.004532432  0.445819274  0.873317289

$boxes$y1
[1]  0.878511331  0.476865573 -0.005109338 -0.005109338 -0.005109338

$boxes$x2
[1] 0.6738183 0.3280558 0.1156127 0.5489874 1.0052157

$boxes$y2
[1] 1.0014382 0.5997924 0.1178175 0.1178175 0.1178175


$split.labs
[1] ""

$split.cex
[1] 1 1 1 1 1

$split.box
$split.box$x1
[1] 0.4642173 0.1315140        NA        NA        NA

$split.box$y1
[1] 0.7959747 0.3943290        NA        NA        NA

$split.box$x2
[1] 0.7515209 0.4214294        NA        NA        NA

$split.box$y2
[1] 0.8662186 0.4645729        NA        NA        NA

#Models

Prep

loan <- loan %>%
  mutate_if(is.character,as.factor) %>%
   mutate(loan_status=case_when(loan_status=="default"~"1",
                              loan_status=="current"~"0")) %>% 
  mutate(loan_status = factor(loan_status))
 

head(loan)

Train Test Split

set.seed(08)

train_test_spit<- initial_split(loan, prop = 0.7, strata=loan_status)

train <- training(train_test_spit)
test  <- testing(train_test_spit)


sprintf("Train PCT : %1.2f%%", nrow(train)/ nrow(loan) * 100)
[1] "Train PCT : 70.00%"
sprintf("Test  PCT : %1.2f%%", nrow(test)/ nrow(loan) * 100)
[1] "Test  PCT : 30.00%"
# Kfold cross validation
kfold_splits <- vfold_cv(train, v=5)

Recipe

# -- define recipe 
model_loan_recipe <- recipe(loan_status ~ loan_amnt + funded_amnt+funded_amnt_inv+term+installment+grade+sub_grade+emp_length+home_ownership+annual_inc+verification_status+issue_d+loan_status+pymnt_plan+purpose+addr_state+dti+delinq_2yrs+fico_range_low+fico_range_high+inq_last_6mths+open_acc+pub_rec+revol_bal+total_acc+out_prncp+out_prncp_inv+total_rec_late_fee+last_pymnt_d+last_pymnt_amnt+last_credit_pull_d+collections_12_mths_ex_med+policy_code+application_type+acc_now_delinq+chargeoff_within_12_mths+delinq_amnt+pub_rec_bankruptcies+tax_liens, 
                      data = train) %>%
  step_unknown(all_nominal_predictors()) %>%
  step_nzv(all_nominal_predictors()) %>%
  step_impute_median(all_numeric_predictors()) %>%
  step_dummy(all_nominal_predictors())

## -- define recipe for an MLP 
loan_recipe_nn <- recipe(loan_status ~ loan_amnt + funded_amnt+funded_amnt_inv+term+installment+grade+sub_grade+emp_length+home_ownership+annual_inc+verification_status+issue_d+loan_status+pymnt_plan+purpose+addr_state+dti+delinq_2yrs+fico_range_low+fico_range_high+inq_last_6mths+open_acc+pub_rec+revol_bal+total_acc+out_prncp+out_prncp_inv+total_rec_late_fee+last_pymnt_d+last_pymnt_amnt+last_credit_pull_d+collections_12_mths_ex_med+policy_code+application_type+acc_now_delinq+chargeoff_within_12_mths+delinq_amnt+pub_rec_bankruptcies+tax_liens, 
                      data = train) %>%
  step_unknown(all_nominal_predictors()) %>%
  themis::step_downsample(loan_status,under_ratio = 1) %>%
  step_nzv(all_nominal_predictors()) %>%
  step_zv(all_predictors()) %>% 
  step_normalize(all_numeric_predictors()) %>% 
  step_impute_median(all_numeric_predictors()) %>%
  step_normalize(all_numeric_predictors())  %>%
  step_dummy(all_nominal_predictors())

bake(loan_recipe_nn %>% prep(), train %>% sample_n(1000))

Models & Workflows

# -- XGB model & workflow 
xgb_model <- boost_tree(
  trees = 20) %>% 
  set_engine("xgboost") %>% 
  set_mode("classification")

xgb_workflow_fit <- workflow() %>%
  add_recipe(model_loan_recipe) %>%
  add_model(xgb_model) %>% 
  fit(train)
[WARNING] Deprecated: --self-contained. use --embed-resources --standalone
# -- RF model & workflow 
rf_model <- rand_forest(
  trees = 20) %>% 
  set_engine("ranger",num.threads = 8, importance = "permutation") %>% 
  set_mode("classification" )

rf_workflow_fit <- workflow() %>%
  add_recipe(model_loan_recipe) %>%
  add_model(rf_model) %>% 
  fit(train)

# -- NNet model & workflow 
nn_model <- mlp(hidden_units = 10, dropout = 0.01, epochs = 20) %>% 
  set_engine("nnet", MaxNWts=10240) %>%
  set_mode("classification")

nn_workflow_fit <- workflow() %>%
  add_recipe(loan_recipe_nn) %>%
  add_model(nn_model) %>% 
  fit(train)

Standard Evaluation

evaluate_models <- function(model_workflow, model_name){
    # 1. Make Predictions
score_train <- bind_cols(
  predict(model_workflow,train, type="prob"), 
  predict(model_workflow,train, type="class"),
  train) %>% 
  mutate(part = "train") 

score_test <- bind_cols(
  predict(model_workflow,test, type="prob"), 
   predict(model_workflow,test, type="class"),
  test) %>% 
  mutate(part = "test") 

options(yardstick.event_first = FALSE)

bind_rows(score_train, score_test) %>%
  group_by(part) %>%
  metrics(loan_status, .pred_1, estimate=.pred_class) %>%
  pivot_wider(id_cols = part, names_from = .metric, values_from = .estimate) %>%
  mutate(model_name = model_name) %>% print()

# ROC Curve 
bind_rows(score_train, score_test) %>%
  group_by(part) %>%
  roc_curve(truth=loan_status, predicted=.pred_1) %>% 
  autoplot() +
   geom_vline(xintercept = 0.20,    
             color = "black",
             linetype = "longdash") +
   labs(title = model_name, x = "FPR(1 - specificity)", y = "TPR(recall)") -> roc_chart 

 
  print(roc_chart)
# Score Distribution 
score_test %>%
  ggplot(aes(.pred_1,fill=loan_status)) +
  geom_histogram(bins=50) +
  geom_vline(aes(xintercept=.5, color="red")) +
  geom_vline(aes(xintercept=.3, color="green")) +
  geom_vline(aes(xintercept=.7, color="blue")) +
  labs(title = model_name) -> score_dist 

print(score_dist)

  # Variable Importance 
  model_workflow %>%
    extract_fit_parsnip() %>%
    vip(10) + 
    labs(model_name)  -> vip_model 
  
    print(vip_model)
    
  
}

evaluate_models(xgb_workflow_fit, "XGB model")

evaluate_models(rf_workflow_fit, "RF model")

evaluate_models(nn_workflow_fit, "NNet model")



scored_test %>% 
  roc_curve(loan_status,.pred_1)  %>% 
  mutate(fpr = 1 - specificity) %>%
  ggplot(aes(x=.threshold,y=sensitivity)) +
  geom_line() + 
  labs(title="Threshold vs TPR", x=".pred_1",y="TPR")





scored_train %>% 
  conf_mat(loan_status,.pred_class) %>% 
    autoplot(type = "heatmap") + 
    labs(title=" training confusion matrix") %>%
    print()
$title
[1] " training confusion matrix"

attr(,"class")
[1] "labels"

scored_test %>% 
  conf_mat(loan_status,.pred_class) %>% 
    autoplot(type = "heatmap") + 
    labs(title=" training confusion matrix") %>%
    print()
$title
[1] " training confusion matrix"

attr(,"class")
[1] "labels"

Global Importance

xgb_workflow_fit %>%
    pull_workflow_fit() %>%
    vip(10) + 
    labs("XGB VIP")  
Warning: `pull_workflow_fit()` was deprecated in workflows 0.2.3.
Please use `extract_fit_parsnip()` instead.

rf_workflow_fit %>%
    pull_workflow_fit() %>%
    vip(10) + 
    labs("RF VIP")  
Warning: `pull_workflow_fit()` was deprecated in workflows 0.2.3.
Please use `extract_fit_parsnip()` instead.

nn_workflow_fit %>%
    pull_workflow_fit() %>%
    vip(10) + 
    labs("NN VIP")  
Warning: `pull_workflow_fit()` was deprecated in workflows 0.2.3.
Please use `extract_fit_parsnip()` instead.

predict(xgb_workflow_fit, kaggle, type = "prob")  %>%
  bind_cols(kaggle) %>%
  dplyr::select(id,loan_status = .pred_1) %>%
  write_csv("my_kaggle.csv")
Warning: Novel levels found in column 'issue_d': 'Dec-2011', 'Nov-2011', 'Oct-2011', 'Sep-2011', 'Aug-2011', 'Jul-2011', 'Jun-2011', 'May-2011', 'Apr-2011', 'Mar-2011', 'Feb-2011', 'Jan-2011', 'Dec-2010', 'Nov-2010', 'Oct-2010', 'Sep-2010', 'Aug-2010', 'Jul-2010', 'Jun-2010', 'May-2010', 'Apr-2010', 'Mar-2010', 'Feb-2010', 'Jan-2010', 'Dec-2009', 'Nov-2009', 'Oct-2009', 'Sep-2009', 'Aug-2009', 'Jul-2009', 'Jun-2009', 'May-2009', 'Apr-2009', 'Mar-2009', 'Feb-2009', 'Jan-2009', 'Dec-2008', 'Nov-2008', 'Oct-2008', 'Sep-2008', 'Aug-2008', 'Jul-2008', 'Jun-2008', 'May-2008', 'Apr-2008', 'Mar-2008', 'Feb-2008', 'Jan-2008', 'Dec-2007', 'Nov-2007', 'Oct-2007', 'Sep-2007', 'Aug-2007', 'Jul-2007', 'Jun-2007'. The levels have been removed, and values have been coerced to 'NA'.Warning: Novel levels found in column 'last_pymnt_d': 'Jun-2014', 'Sep-2016', 'Jan-2015', 'Nov-2012', 'Jul-2012', 'Feb-2015', 'Oct-2013', 'Oct-2012', 'Sep-2012', 'Apr-2013', 'Oct-2014', 'Aug-2012', 'Jul-2013', 'Jan-2016', 'Feb-2016', 'Aug-2013', 'Apr-2015', 'Apr-2014', 'Jun-2012', 'Feb-2013', 'Jun-2013', 'Nov-2013', 'Sep-2014', 'Dec-2014', 'Mar-2013', 'Dec-2013', 'Mar-2014', 'May-2014', 'Nov-2015', 'Jan-2014', 'May-2016', 'Jun-2016', 'Dec-2012', 'Feb-2012', 'Feb-2014', 'Aug-2015', 'Jul-2016', 'Jul-2014', 'Jan-2013', 'May-2013', 'Apr-2012', 'Mar-2016', 'Sep-2013', 'Sep-2015', 'Aug-2014', 'Jun-2015', 'Nov-2014', 'Aug-2016', 'Dec-2015', 'Jul-2015', 'Apr-2016', 'Jan-2012', 'May-2015', 'May-2012', 'Mar-2012', 'Oct-2015', 'Mar-2015', 'Dec-2011', 'Nov-2011', 'Oct-2011', 'Sep-2011', 'Aug-2011', 'Jul-2011', 'Jun-2011', 'May-2011', 'Apr-2011', 'Mar-2011', 'Feb-2011', 'Jan-2011', 'Dec-2010', 'Nov-2010', 'Oct-2010', 'Sep-2010', 'Aug-2010', 'Jul-2010', 'Jun-2010', 'May-2010', 'Apr-2010', 'Mar-2010', 'Feb-2010', 'Jan-2010', 'Dec-2009', 'Nov-2009', 'Oct-2009', 'Aug-2009', 'Jul-2009', 'Sep-2009', 'May-2009', 'Jun-2009', 'Apr-2009', 'Mar-2009', 'Jan-2009', 'Feb-2009', 'Dec-2008', 'Jul-2008', 'Oct-2008', 'Jun-2008', 'Nov-2008', 'Aug-2008', 'Apr-2008', 'May-2008', 'Mar-2008', 'Sep-2008', 'Dec-2007', 'Jan-2008'. The levels have been removed, and values have been coerced to 'NA'.Warning: Novel levels found in column 'last_credit_pull_d': 'Sep-2016', 'Jan-2016', 'Apr-2015', 'Jul-2015', 'Feb-2016', 'Mar-2014', 'Sep-2012', 'Dec-2014', 'Jun-2012', 'Mar-2015', 'Sep-2014', 'Apr-2014', 'Oct-2014', 'Feb-2013', 'Nov-2015', 'Oct-2012', 'Nov-2013', 'Nov-2014', 'Jul-2016', 'Oct-2015', 'Jan-2015', 'Aug-2015', 'Aug-2012', 'Sep-2013', 'Aug-2014', 'Jun-2016', 'Feb-2012', 'Oct-2013', 'Jan-2014', 'Jun-2013', 'Dec-2015', 'Jul-2014', 'Mar-2016', 'Dec-2013', 'Apr-2016', 'Sep-2015', 'Apr-2013', 'Nov-2012', 'May-2014', 'May-2015', 'Jun-2014', 'Jul-2012', 'May-2013', 'Feb-2015', 'Aug-2016', 'Jun-2015', 'Jan-2012', 'May-2016', 'Jul-2013', 'Feb-2014', 'Mar-2013', 'Mar-2012', 'Aug-2013', 'May-2012', 'Dec-2012', 'Dec-2011', 'Apr-2012', 'Oct-2011', 'Nov-2011', 'Aug-2011', 'Jan-2013', 'Sep-2011', 'Jul-2011', 'Jun-2011', 'May-2011', 'Apr-2011', 'Mar-2011', 'Feb-2011', 'Jan-2011', 'Dec-2010', 'Nov-2010', 'Oct-2010', 'Sep-2010', 'Aug-2010', 'Jul-2010', 'Jun-2010', 'Apr-2010', 'Mar-2010', 'Feb-2010', 'Aug-2007', 'May-2010', 'Dec-2009', 'Jan-2010', 'Nov-2009', 'Oct-2009', 'Jul-2009', 'Aug-2009', 'Apr-2009', 'Jun-2009', 'Feb-2009', 'Dec-2008', 'May-2009', 'May-2008', 'Mar-2009', 'Mar-2008', 'Sep-2008', 'Sep-2009', 'Feb-2008', 'Oct-2008', 'Jan-2009', 'Jan-2008', 'Oct-2007', 'Jun-2007', 'Aug-2008', 'Nov-2007', 'Sep-2007', 'May-2007'. The levels have been removed, and values have been coerced to 'NA'.
xgb_workflow_fit %>% 
  pull_workflow_fit() %>%
  vip(10)
Warning: `pull_workflow_fit()` was deprecated in workflows 0.2.3.
Please use `extract_fit_parsnip()` instead.

xgb_explainer <- explain_tidymodels(
  xgb_workflow_fit,
  data = train ,
  y = train$loan_default ,
  verbose = TRUE
)
Preparation of a new explainer is initiated
  -> model label       :  workflow  (  default  )
  -> data              :  20843  rows  52  cols 
  -> data              :  tibble converted into a data.frame 
Warning: Unknown or uninitialised column: `loan_default`.
  -> target variable   :  not specified! (  WARNING  )
  -> predict function  :  yhat.workflow  will be used (  default  )
  -> predicted values  :  No value for predict function target column. (  default  )
  -> model_info        :  package tidymodels , ver. 1.0.0 , task classification (  default  ) 
  -> model_info        :  Model info detected classification task but 'y' is a NULL .  (  WARNING  )
  -> model_info        :  By deafult classification tasks supports only numercical 'y' parameter. 
  -> model_info        :  Consider changing to numerical vector with 0 and 1 values.
  -> model_info        :  Otherwise I will not be able to calculate residuals or loss function.
  -> predicted values  :  numerical, min =  0.00114727 , mean =  0.1510182 , max =  0.9787124  
  -> residual function :  difference between y and yhat (  default  )
  A new explainer has been created!  
pdp_grade <- model_profile(
  xgb_explainer,
  variables = c("grade")
)
'variable_type' changed to 'categorical' due to lack of numerical variables.
plot(pdp_grade) + 
  labs(title = "PDP loan GRADE", 
       x="grade", 
       y="average impact on prediction") 

  
  
as_tibble(pdp_grade$agr_profiles) %>%
  mutate(profile_variable = `_x_`,
         avg_prediction_impact = `_yhat_`) %>%
  ggplot(aes(x=profile_variable, y=avg_prediction_impact)) +
  geom_col() +
  labs(
    x = "Variable: Loan GRADE",
     y = " Average prediction Impact ",
    color = NULL,
    title = "Partial dependence plot Loan GRADE",
    subtitle = "How does GRADE impact predictions (on average)"
  ) 


pdp_fico <- model_profile(
  xgb_explainer,
  variables = c("fico_range_low")
)

plot(pdp_fico)



pdp_income <- model_profile(
  xgb_explainer,
  variables = c("annual_inc")
)

plot(pdp_income)



as_tibble(pdp_fico$agr_profiles) %>%
  mutate(profile_variable = `_x_`,
         avg_prediction_impact = `_yhat_`) %>%
  ggplot(aes(x=profile_variable, y=avg_prediction_impact)) +
  geom_line() +
  labs(
    x = "Variable: Fico Range Low",
     y = " Average prediction Impact ",
    color = NULL,
    title = "Partial dependence plot Loan GRADE",
    subtitle = "How does Fico Range Low impact predictions (on average)"
  ) 


as_tibble(pdp_income$agr_profiles) %>%
  mutate(profile_variable = `_x_`,
         avg_prediction_impact = `_yhat_`) %>%
  filter(profile_variable < 6000000) %>%
  ggplot(aes(x=profile_variable, y=avg_prediction_impact)) +
  geom_line() +
  labs(
    x = "Variable: Fico Range Low",
     y = " Average prediction Impact ",
    color = NULL,
    title = "Partial dependence plot Loan GRADE",
    subtitle = "How does Fico Range Low impact predictions (on average)"
  ) 

library(DALEX)
library(DALEXtra)

xgb_explainer <- explain_tidymodels(
  xgb_workflow_fit,
  data = train ,
  y = train$loan_status ,
  verbose = TRUE
)
Preparation of a new explainer is initiated
  -> model label       :  workflow  (  default  )
  -> data              :  20843  rows  52  cols 
  -> data              :  tibble converted into a data.frame 
  -> target variable   :  20843  values 
  -> predict function  :  yhat.workflow  will be used (  default  )
  -> predicted values  :  No value for predict function target column. (  default  )
  -> model_info        :  package tidymodels , ver. 1.0.0 , task classification (  default  ) 
  -> model_info        :  Model info detected classification task but 'y' is a factor .  (  WARNING  )
  -> model_info        :  By deafult classification tasks supports only numercical 'y' parameter. 
  -> model_info        :  Consider changing to numerical vector with 0 and 1 values.
  -> model_info        :  Otherwise I will not be able to calculate residuals or loss function.
  -> predicted values  :  numerical, min =  0.00114727 , mean =  0.1510182 , max =  0.9787124  
  -> residual function :  difference between y and yhat (  default  )
Warning: ‘-’ not meaningful for factors
  -> residuals         :  numerical, min =  NA , mean =  NA , max =  NA  
  A new explainer has been created!  
pdp_age <- model_profile(
  xgb_explainer,
  variables = "annual_inc"
)


pdp_income <- model_profile(
  xgb_explainer,
  variables = "annual_inc"
)

plot(pdp_income)

  labs(title = "PDP annual_inc", x="annual_inc", y="average impact on prediction") 
$x
[1] "annual_inc"

$y
[1] "average impact on prediction"

$title
[1] "PDP annual_inc"

attr(,"class")
[1] "labels"
  
  as_tibble(pdp_income$agr_profiles) %>%
  mutate(profile_variable = `_x_`,
         avg_prediction_impact = `_yhat_`) %>%
    filter(profile_variable < 6000000.00) %>%
  ggplot(aes(x=profile_variable, y=avg_prediction_impact)) +
  geom_line() +
  labs(
    x = "Variable: annual_inc",
     y = " Average prediction Impact ",
    color = NULL,
    title = "Partial dependence plot Loan GRADE",
    subtitle = "How does annual_inc impact predictions (on average)"
  ) 

Prediction Explainer

# speed things up! 
train_sample <- train %>% 
  select(last_credit_pull_d, # select just the columns used in recipe 
         last_pymnt_amnt,
         term,
         last_pymnt_d,
         total_rec_late_fee,
         installment,
         annual_inc,
         inq_last_6mths,
         funded_amnt_inv) %>%
  sample_frac(0.1) # take a 10% sample or less

xgb_explainer <- explain_tidymodels(
  xgb_workflow_fit,
  data = train_sample ,
  y = train_sample$loan_status ,
  verbose = TRUE
)
Preparation of a new explainer is initiated
  -> model label       :  workflow  (  default  )
  -> data              :  2084  rows  9  cols 
  -> data              :  tibble converted into a data.frame 
Warning: Unknown or uninitialised column: `loan_status`.
  -> target variable   :  not specified! (  WARNING  )
  -> predict function  :  yhat.workflow  will be used (  default  )
  -> predicted values  :  No value for predict function target column. (  default  )
  -> model_info        :  package tidymodels , ver. 1.0.0 , task classification (  default  ) 
  -> model_info        :  Model info detected classification task but 'y' is a NULL .  (  WARNING  )
  -> model_info        :  By deafult classification tasks supports only numercical 'y' parameter. 
  -> model_info        :  Consider changing to numerical vector with 0 and 1 values.
  -> model_info        :  Otherwise I will not be able to calculate residuals or loss function.
  -> predicted values  :  the predict_function returns an error when executed (  WARNING  ) 
  -> residual function :  difference between y and yhat (  default  )
  A new explainer has been created!  
# you should use TEST not training for this! 
score_test %>% head()

# Top 5 TP highest scoring defaults 
top_5_tp <- score_test %>%
  filter(.pred_class == loan_status) %>%
  filter(loan_status != 1) %>%
  slice_max(order_by = .pred_1, n=5)

# Top 5 FP highest scoring defaults 
top_5_fp <- score_test %>%
  filter(.pred_class == loan_status) %>%
  filter(loan_status != 1) %>%
  slice_max(order_by = .pred_1, n=5)

# Bottom 5 FN lowest scoring defaults 
bottom_5_fn <- score_test %>%
  filter(.pred_class == loan_status) %>%
  filter(loan_status == 1) %>%
  slice_min(order_by = .pred_1, n=5)

Local Explainer



explain_prediction <- function(top_5_tp){
# step 1. run the explainer 
record_shap <- predict_parts(explainer = xgb_explainer, 
                               new_observation = top_5_tp,
                               type="shap")

# step 2. get a predicted probability for plot 
prediction_prob <- top_5_tp[,".pred_1"] %>% 
  mutate(.pred_default = round(.pred_1,3)) %>% 
  pull() 

# step 3. plot it. 
# you notice you don't get categorical values ...  
record_shap %>% 
  plot() +
  labs(title=paste("SHAP Explainer:",prediction_prob),
       x = "shap importance",
       y = "record") -> shap_plot 

print(shap_plot)
}

# example TP 5 records
for (row in 1:nrow(top_5_tp)) {
    s_record <- top_5_tp[row,]
    explain_prediction(s_record)
} 
loan_sample <- train %>% sample_n(1000)
loans_explainer <- explain_tidymodels(
    xgb_workflow_fit,   # fitted workflow object 
    data = loan_sample,    # original training data
    y = loan_sample$loan_status, # predicted outcome 
    label = "xgboost",
    verbose = FALSE
  )
Warning: ‘-’ not meaningful for factors
explain_prediction <- function(single_record){
  # step 3. run the explainer 
record_shap <- predict_parts(explainer = loans_explainer, 
                               new_observation = single_record,
                               #type="fastshap"
                             )

# step 4. plot it. 
# you notice you don't get categorical values ...  
record_shap %>% plot() %>% print()

# --- more involved explanations with categories. ---- 

# step 4a.. convert breakdown to a tibble so we can join it
record_shap %>%
  as_tibble() -> shap_data 

# step 4b. transpose your single record prediction 
single_record %>% 
 gather(key="variable_name",value="value") -> prediction_data 

# step 4c. get a predicted probability for plot 
prediction_prob <- single_record[,".pred_1"] %>% mutate(.pred_1 = round(.pred_1,3)) %>% pull() 

# step 5. plot it.
shap_data %>% 
  inner_join(prediction_data) %>%
  mutate(variable = paste(variable_name,value,sep = ": ")) %>% 
  group_by(variable) %>%
  summarize(contribution = mean(contribution)) %>%
  mutate(contribution = round(contribution,3),
         sign = if_else(contribution < 0, "neg","pos")) %>%
  ggplot(aes(y=reorder(variable, contribution), x= contribution, fill=sign)) +
  geom_col() + 
  geom_text(aes(label=contribution))+
  labs(
    title = "SHAPLEY explainations",
    subtitle = paste("predicted probablity = ",prediction_prob) ,
                    x="contribution",
                    y="features")
  
}

 # -- score training 
scored_train <- predict(xgb_workflow_fit, train, type="prob") %>%
  bind_cols(predict(xgb_workflow_fit, train, type="class")) %>%
  bind_cols(.,train) 

# -- score testing 
scored_test <- predict(xgb_workflow_fit, test, type="prob") %>%
  bind_cols(predict(xgb_workflow_fit, test, type="class")) %>%
  bind_cols(.,test)
  
  
top_5_tp <- scored_test %>%
  filter(.pred_class == loan_status) %>%
  filter(loan_status == 1) %>%
  slice_max(.pred_1,n=5)

top_5_fp <- scored_test %>%
  filter(.pred_class != loan_status) %>%
  filter(loan_status == 1) %>%
  slice_max(.pred_1,n=5)

top_5_fn <- scored_test %>%
  filter(.pred_class != loan_status ) %>%
  filter(loan_status == 1) %>%
  slice_max(.pred_1,n=5)


# repeat for FP and FN 
for (row in 1:nrow(top_5_tp)) {
    s_record <- top_5_tp[row,]
    explain_prediction(s_record)
} 
[WARNING] Deprecated: --self-contained. use --embed-resources --standalone

for (row in 1:nrow(top_5_fp)) {
    s_record <- top_5_tp[row,]
    explain_prediction(s_record)
} 


for (row in 1:nrow(top_5_fn)) {
    s_record <- top_5_tp[row,]
    explain_prediction(s_record)
} 

LS0tDQp0aXRsZTogIklzb2xhdGlvbiBGb3Jlc3RzIg0Kb3V0cHV0OiBodG1sX25vdGVib29rDQotLS0NCg0KIyBJbXBvcnQgbGlicmFyaWVzIA0KDQpgYGB7cn0NCmxpYnJhcnkodGlkeXZlcnNlKQ0KbGlicmFyeSh0aWR5bW9kZWxzKQ0KbGlicmFyeShzb2xpdHVkZSkgIyAtLSBuZXcgcGFja2FnZSANCmxpYnJhcnkoamFuaXRvcikNCmxpYnJhcnkoZ2dwdWJyKQ0KbGlicmFyeShza2ltcikNCmxpYnJhcnkodGhlbWlzKQ0KbGlicmFyeShkcGx5cikNCmxpYnJhcnkodmlwKQ0KbGlicmFyeShEQUxFWCkgICAgIyBuZXcgDQpsaWJyYXJ5KERBTEVYdHJhKSAjIG5ldw0KbGlicmFyeShycGFydCkNCmxpYnJhcnkocnBhcnQucGxvdCkNCmBgYA0KDQoNCmBgYHtyfQ0KbG9hbiA8LSByZWFkX2NzdigibG9hbl90cmFpbi5jc3YiKSAlPiUNCiAgY2xlYW5fbmFtZXMoKQ0KDQpoZWFkKGxvYW4pDQpza2ltKGxvYW4pDQoNCmthZ2dsZSA8LSByZWFkX2NzdigibG9hbl9ob2xkb3V0LmNzdiIpICU+JSBjbGVhbl9uYW1lcygpDQpza2ltKGthZ2dsZSkNCmBgYA0KDQoNCiMgZXhwbG9yZSANCg0KYGBge3J9DQpuX2NvbHMgPC0gbmFtZXMobG9hbiAlPiUgc2VsZWN0X2lmKGlzLm51bWVyaWMpICU+JSBzZWxlY3QoLWlkLC1tZW1iZXJfaWQpKQ0KDQpteV9oaXN0IDwtIGZ1bmN0aW9uKGNvbCl7DQogIGxvYW4gJT4lDQogICAgc3VtbWFyaXNlKG49bigpLCANCiAgICAgICAgICAgICAgbl9taXNzID0gc3VtKGlzLm5hKCEhYXMubmFtZShjb2wpKSksDQogICAgICAgICAgICAgIG5fZGlzdCA9IG5fZGlzdGluY3QoISFhcy5uYW1lKGNvbCkpLA0KICAgICAgICAgICAgICBtZWFuID0gcm91bmQobWVhbighIWFzLm5hbWUoY29sKSwgbmEucm09VFJVRSksMiksDQogICAgICAgICAgICAgIG1pbiAgPSBtaW4oISFhcy5uYW1lKGNvbCksIG5hLnJtPVRSVUUpLA0KICAgICAgICAgICAgICBtYXggID0gbWF4KCEhYXMubmFtZShjb2wpLCBuYS5ybT1UUlVFKQ0KICAgICAgICAgICAgICApIC0+IGNvbF9zdW1tYXJ5DQogIA0KICAgcDEgIDwtIGdndGV4dHRhYmxlKGNvbF9zdW1tYXJ5LCByb3dzID0gTlVMTCwgDQogICAgICAgICAgICAgICAgICAgICAgICB0aGVtZSA9IHR0aGVtZSgibU9yYW5nZSIpKQ0KICANCmgxIDwtIGxvYW4gJT4lDQogIGdncGxvdChhZXMoeD0hIWFzLm5hbWUoY29sKSkpICsNCiAgZ2VvbV9oaXN0b2dyYW0oYmlucz0zMCkgDQoNCnBsdCA8LSBnZ2FycmFuZ2UoIGgxLCBwMSwgDQogICAgICAgICAgbmNvbCA9IDEsIG5yb3cgPSAyLA0KICAgICAgICAgIGhlaWdodHMgPSBjKDEsIDAuMykpIA0KDQpwcmludChwbHQpDQoNCn0NCg0KZm9yIChjIGluIG5fY29scyl7DQogIG15X2hpc3QoYykNCn0NCmBgYA0KIyBleHBsb3JlIHRhcmdldA0KYGBge3J9DQpsb2FuX3N1bW1hcnkgPC0gbG9hbiAlPiUNCiAgY291bnQobG9hbl9zdGF0dXMpICU+JQ0KICBtdXRhdGUocGN0ID0gbi9zdW0obikpDQoNCg0KbG9hbl9zdW1tYXJ5ICU+JQ0KICBnZ3Bsb3QoYWVzKHg9ZmFjdG9yKGxvYW5fc3RhdHVzKSx5PXBjdCkpICsNCiAgZ2VvbV9jb2woKSAgKyANCiAgZ2VvbV90ZXh0KGFlcyhsYWJlbCA9IHJvdW5kKHBjdCoxMDAsMikpICwgdmp1c3QgPSAyLjUsIGNvbG91ciA9ICJ3aGl0ZSIpICsgDQogIGxhYnModGl0bGU9IkxvYW4gU3RhdHVzIFBsb3QiLCB4PSJMb2FuX1N0YXR1cyIsIHk9IlBDVCIpDQoNCg0KYGBgDQoNCiMgRXhwbG90YXJ5IEFuYWx5c2lzDQoNCmBgYHtyfQ0KbG9hbl92aXMgPC0gbG9hbiAlPiUgDQogICBtdXRhdGVfaWYoaXMuY2hhcmFjdGVyLCBmYWN0b3IpIA0KDQpmb3IgKGMgaW4gbmFtZXMobG9hbl92aXMgJT4lIGRwbHlyOjpzZWxlY3QoIWMoaWQsbWVtYmVyX2lkKSkpKSB7DQogIGlmIChjID09ICJldmVudF90aW1lc3RhbXAiKSB7DQogICAjIHByaW50KCBmcmF1ZF92aXMgJT4lDQogICAgICAgICAgICAgI2dncGxvdCguLCBhZXMoISFhcy5uYW1lKGMpKSkgKyANCiAgICAgICAgICAgICAjZ2VvbV9oaXN0b2dyYW0oYWVzKGJpbnM9MTAsZmlsbCA9IGxvYW5fc3RhdHVzKSwgcG9zaXRpb24gPSAiZmlsbCIpICArbGFicyh0aXRsZSA9IGMsIHkgPSAicGN0IGZyYXVkIikpDQogICAgICANCiAgfWVsc2UgaWYgKGMgJWluJSBuYW1lcyhsb2FuX3ZpcyAlPiUgZHBseXI6OnNlbGVjdCh3aGVyZShpcy5mYWN0b3IpKSkpIHsNCiAgICAjIC0tIGZvciBlYWNoIGNoYXJhY3RlciBjb2x1bW4gY3JlYXRlIGEgY2hhcnQNCiAgICBwcmludCggbG9hbl92aXMgJT4lDQogICAgICAgICAgICAgZ2dwbG90KC4sIGFlcyghIWFzLm5hbWUoYykpKSArIA0KICAgICAgICAgICAgIGdlb21fYmFyKGFlcyhmaWxsID0gbG9hbl9zdGF0dXMpLCBwb3NpdGlvbiA9ICJmaWxsIikgICsgbGFicyh0aXRsZSA9IGMsIHkgPSAicGN0IGZyYXVkIikpDQogIH0gZWxzZSB7DQogICAgIyAtLSBjb21wYXJhdGl2ZSBib3hwbG90cw0KICAgIHByaW50KGdncGxvdChsb2FuX3ZpcywgYWVzKHg9bG9hbl9zdGF0dXMsIHk9ISFhcy5uYW1lKGMpLCBmaWxsPWxvYW5fc3RhdHVzKSkrIGdlb21fYm94cGxvdCgpICtsYWJzKHRpdGxlID0gYykpDQogIH0NCn0NCg0KDQoNCmBgYA0KI2NvcnJlbGF0aW9uDQoNCmBgYHtyfQ0KbGlicmFyeShyZXNoYXBlMikNCmxvYW5fbnVtZXJpYyA8LSBzdWJzZXQobG9hbixzZWxlY3QgPSAtYyhpZCxtZW1iZXJfaWQpKSAlPiUNCiAgc2VsZWN0X2lmKC4saXMubnVtZXJpYykNCg0KY29yX21hdCA8LSBsb2FuX251bWVyaWMgJT4lDQogIGNvcigpDQoNCg0KY29yX21lbHQgPC0gY29yX21hdCAlPiUgbWVsdCANCg0KY29yX21lbHQgJT4lDQogIG11dGF0ZSh2YWx1ZSA9IHJvdW5kKHZhbHVlLDIpKSAlPiUNCiBnZ3Bsb3QoYWVzKFZhcjIsIFZhcjEsIGZpbGwgPSB2YWx1ZSkpKw0KIGdlb21fdGlsZSgpICsNCiBzY2FsZV9maWxsX2dyYWRpZW50Mihsb3cgPSAiYmx1ZSIsIGhpZ2ggPSAicmVkIiwgbWlkID0gIndoaXRlIiwgDQogICAgICAgICAgICAgICAgICAgICAgbWlkcG9pbnQgPSAwLCBsaW1pdCA9IGMoLTEsMSksIHNwYWNlID0gIkxhYiIsIA0KICAgICAgICAgICAgICAgICAgICAgIG5hbWU9IkNvcnJlbGF0aW9uIikgKw0KIHRoZW1lX21pbmltYWwoKSArDQogdGhlbWUoYXhpcy50ZXh0LnggPSBlbGVtZW50X3RleHQoYW5nbGUgPSA0NSwgdmp1c3QgPSAxLCANCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzaXplID0gNCwgaGp1c3QgPSAxKSxheGlzLnRleHQueSA9IGVsZW1lbnRfdGV4dChhbmdsZSA9IDQ1LCB2anVzdCA9IDEsIA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHNpemUgPSA0LCBoanVzdCA9IDEpKSsNCiBjb29yZF9maXhlZCgpICsNCiBnZW9tX3RleHQoYWVzKFZhcjIsIFZhcjEsIGxhYmVsID0gdmFsdWUpLCBjb2xvciA9ICJibGFjayIsIHNpemUgPSAxLjUpICsNCiAgbGFicyh0aXRsZSA9ICJQZWFyc29uIENvcnJlbGF0aW9uIGZvciBOdW1lcmljYWwgRGF0YSIpDQoNCg0KbG9hbl9udW1lcmljICU+JQ0KICBnZ3Bsb3QoYWVzKHg9aW50X3JhdGUseT1maWNvX3JhbmdlX2hpZ2gsY29sb3I9bG9hbiRsb2FuX3N0YXR1cykpICsgZ2VvbV9wb2ludCgpICsgbGFicyh0aXRsZSA9ICJpbnRfcmF0ZSB2cyBmaWNvX3JhbmdlX2hpZ2giKQ0KDQpsb2FuX251bWVyaWMgJT4lDQogIGdncGxvdChhZXMoeD1sYXN0X3B5bW50X2FtbnQseT1maWNvX3JhbmdlX2xvdyxjb2xvcj1sb2FuJGxvYW5fc3RhdHVzKSkgKyBnZW9tX3BvaW50KCkgKyBsYWJzKHRpdGxlID0gImxhc3RfcHltbnRfYW1udCB2cyBmaWNvX3JhbmdlX2xvdyIpDQoNCmBgYA0KDQoNCiMgUmVjaXBlIA0KYGBge3J9DQojIGRlYWwgdy4gY2F0ZWdvcmljYWxzIA0KbG9hbl9yZWNpcGUgPC0gcmVjaXBlKH4uLGxvYW4pICU+JQ0KICBzdGVwX3JtKGlkLG1lbWJlcl9pZCxpbnRfcmF0ZSxlbXBfdGl0bGUsdXJsLGRlc2MsdGl0bGUsemlwX2NvZGUsZWFybGllc3RfY3JfbGluZSxyZXZvbF91dGlsLG10aHNfc2luY2VfbGFzdF9kZWxpbnEsbXRoc19zaW5jZV9sYXN0X3JlY29yZCxuZXh0X3B5bW50X2QpICU+JQ0KICBzdGVwX3Vua25vd24oYWxsX25vbWluYWxfcHJlZGljdG9ycygpKSAlPiUNCiAgc3RlcF9pbXB1dGVfbWVkaWFuKGFsbF9udW1lcmljX3ByZWRpY3RvcnMoKSkgJT4lDQogIHN0ZXBfZHVtbXkoYWxsX25vbWluYWxfcHJlZGljdG9ycygpKSAlPiUNCiAgcHJlcCgpDQoNCmJha2VfbG9hbiA8LSBiYWtlKGxvYW5fcmVjaXBlLCBsb2FuKQ0KDQoNCmBgYA0KDQojIyBUcmFpbiB5b3VyIElzb2xhdGlvbkZvcmVzdA0KYGBge3J9DQppc29fZm9yZXN0IDwtIGlzb2xhdGlvbkZvcmVzdCRuZXcoDQogIHNhbXBsZV9zaXplID0gMTAwMCwNCiAgbnVtX3RyZWVzID0gMTAwLA0KICBtYXhfZGVwdGggPSBjZWlsaW5nKGxvZzIoMTAwMCkpKQ0KDQoNCmlzb19mb3Jlc3QkZml0KGJha2VfbG9hbikNCmBgYA0KDQojIHByZWRpY3QgdHJhaW5pbmcgDQoNCmV2YWx1YXRlIGhpc3RvZ3JhbSBwaWNrIGEgdmFsdWUgb2YgYXZlcmFnZV9kZXB0aCB0byBpZGVudGlmeSBhbm9tYWxpZXMuIGEgc2hvcnRlciBhdmVyYWdlIGRlcHRoIG1lYW5zIHRoZSBwb2ludCBpcyBtb3JlIGlzb2xhdGVkIGFuZCBtb3JlIGxpa2VseSBhbiBhbm9tYWx5IA0KDQpgYGB7cn0NCnByZWRfdHJhaW4gPC0gaXNvX2ZvcmVzdCRwcmVkaWN0KGJha2VfbG9hbikNCg0KcHJlZF90cmFpbiAlPiUNCiAgZ2dwbG90KGFlcyhhdmVyYWdlX2RlcHRoKSkgKw0KICBnZW9tX2hpc3RvZ3JhbShiaW5zPTIwKSArIA0KICBnZW9tX3ZsaW5lKHhpbnRlcmNlcHQgPSA5LjQ1LCBsaW5ldHlwZT0iZG90dGVkIiwgDQogICAgICAgICAgICAgICAgY29sb3IgPSAiYmx1ZSIsIHNpemU9MS41KSArIA0KICBsYWJzKHRpdGxlPSJJc29sYXRpb24gRm9yZXN0IEF2ZXJhZ2UgVHJlZSBEZXB0aCIpDQoNCnByZWRfdHJhaW4gJT4lDQogIGdncGxvdChhZXMoYW5vbWFseV9zY29yZSkpICsNCiAgZ2VvbV9oaXN0b2dyYW0oYmlucz0yMCkgKyANCiAgZ2VvbV92bGluZSh4aW50ZXJjZXB0ID0gMC42LCBsaW5ldHlwZT0iZG90dGVkIiwgDQogICAgICAgICAgICAgICAgY29sb3IgPSAiYmx1ZSIsIHNpemU9MS41KSArIA0KICBsYWJzKHRpdGxlPSJJc29sYXRpb24gRm9yZXN0IEFub21hbHkgU2NvcmUgQWJvdmUgMC42IikNCg0KDQpgYGANCg0KIyBnbG9iYWwgbGV2ZWwgaW50ZXJwcmV0YXRpb24gDQoNClRoZSBzdGVwcyBvZiBpbnRlcnByZXRpbmcgYW5vbWFsaWVzIG9uIGEgZ2xvYmFsIGxldmVsIGFyZToNCg0KMS4gQ3JlYXRlIGEgZGF0YSBmcmFtZSB3aXRoIGEgY29sdW1uIHRoYXQgaW5kaWNhdGVzIHdoZXRoZXIgdGhlIHJlY29yZCB3YXMgY29uc2lkZXJlZCBhbiBhbm9tYWx5Lg0KMi4gVHJhaW4gYSBkZWNpc2lvbiB0cmVlIHRvIHByZWRpY3QgdGhlIGFub21hbHkgZmxhZy4NCjMuIFZpc3VhbGl6ZSB0aGUgZGVjaXNpb24gdHJlZSB0byBkZXRlcm1pbmUgd2hpY2ggc2VnbWVudHMgb2YgdGhlIGRhdGEgYXJlIGNvbnNpZGVyZWQgYW5vbWFsb3VzLg0KDQpgYGB7cn0NCnRyYWluX3ByZWQgPC0gYmluZF9jb2xzKGlzb19mb3Jlc3QkcHJlZGljdChiYWtlX2xvYW4pLGJha2VfbG9hbikgJT4lDQogIG11dGF0ZShhbm9tYWx5ID0gYXMuZmFjdG9yKGlmX2Vsc2UoYXZlcmFnZV9kZXB0aCA8PSA5LjQ1LCAiQW5vbWFseSIsIk5vcm1hbCIpKSkNCg0KdHJhaW5fcHJlZCAlPiUNCiAgYXJyYW5nZShhdmVyYWdlX2RlcHRoKSAlPiUNCiAgY291bnQoYW5vbWFseSkNCg0KYGBgDQoNCiMjIEZpdCBhIFRyZWUgDQpgYGB7cn0NCmZtbGEgPC0gYXMuZm9ybXVsYShwYXN0ZSgiYW5vbWFseSB+ICIsIHBhc3RlKGJha2VfbG9hbiAlPiUgY29sbmFtZXMoKSwgY29sbGFwc2U9ICIrIikpKQ0KDQpvdXRsaWVyX3RyZWUgPC0gZGVjaXNpb25fdHJlZShtaW5fbj0yLCB0cmVlX2RlcHRoPTMsIGNvc3RfY29tcGxleGl0eSA9IC4wMSkgJT4lDQogIHNldF9tb2RlKCJjbGFzc2lmaWNhdGlvbiIpICU+JQ0KICBzZXRfZW5naW5lKCJycGFydCIpICU+JQ0KICBmaXQoZm1sYSwgZGF0YT10cmFpbl9wcmVkKQ0KDQpvdXRsaWVyX3RyZWUkZml0DQpgYGANCg0KYGBge3J9DQpsaWJyYXJ5KHJwYXJ0LnBsb3QpICMgLS0gcGxvdHRpbmcgZGVjaXNpb24gdHJlZXMgDQoNCnJwYXJ0LnBsb3Qob3V0bGllcl90cmVlJGZpdCxjbGlwLnJpZ2h0LmxhYnMgPSBGQUxTRSwgYnJhbmNoID0gLjMsIHVuZGVyID0gVFJVRSwgcm91bmRpbnQ9RkFMU0UsIGV4dHJhPTMpDQoNCmBgYA0KIyBHbG9iYWwgQW5vbWFseSBSdWxlcyANCg0KYGBge3J9DQphbm9tYWx5X3J1bGVzIDwtIHJwYXJ0LnJ1bGVzKG91dGxpZXJfdHJlZSRmaXQscm91bmRpbnQ9RkFMU0UsIGV4dHJhID0gNCwgY292ZXIgPSBUUlVFLCBjbGlwLmZhY3MgPSBUUlVFKSAlPiUgY2xlYW5fbmFtZXMoKSAlPiUNCiAgI2ZpbHRlcihhbm9tYWx5PT0iQW5vbWFseSIpICU+JQ0KICBtdXRhdGUocnVsZSA9ICJJRiIpIA0KDQoNCnJ1bGVfY29scyA8LSBhbm9tYWx5X3J1bGVzICU+JSBzZWxlY3Qoc3RhcnRzX3dpdGgoInhfIikpICU+JSBjb2xuYW1lcygpDQoNCmZvciAoY29sIGluIHJ1bGVfY29scyl7DQphbm9tYWx5X3J1bGVzIDwtIGFub21hbHlfcnVsZXMgJT4lDQogICAgbXV0YXRlKHJ1bGUgPSBwYXN0ZShydWxlLCAhIWFzLm5hbWUoY29sKSkpDQp9DQoNCmFub21hbHlfcnVsZXMgJT4lDQogIGFzLmRhdGEuZnJhbWUoKSAlPiUNCiAgZmlsdGVyKGFub21hbHkgPT0gIkFub21hbHkiKSAlPiUNCiAgbXV0YXRlKHJ1bGUgPSBwYXN0ZShydWxlLCAiIFRIRU4gIiwgYW5vbWFseSApKSAlPiUNCiAgbXV0YXRlKHJ1bGUgPSBwYXN0ZShydWxlLCIgY292ZXJhZ2UgIiwgY292ZXIpKSAlPiUNCiAgc2VsZWN0KCBydWxlKQ0KDQphbm9tYWx5X3J1bGVzICU+JQ0KICBhcy5kYXRhLmZyYW1lKCkgJT4lDQogIGZpbHRlcihhbm9tYWx5ID09ICJOb3JtYWwiKSAlPiUNCiAgbXV0YXRlKHJ1bGUgPSBwYXN0ZShydWxlLCAiIFRIRU4gIiwgYW5vbWFseSApKSAlPiUNCiAgbXV0YXRlKHJ1bGUgPSBwYXN0ZShydWxlLCIgY292ZXJhZ2UgIiwgY292ZXIpKSAlPiUNCiAgc2VsZWN0KCBydWxlKQ0KYGBgDQoNCmBgYHtyfQ0KDQpwcmVkX3RyYWluIDwtIGJpbmRfY29scyhpc29fZm9yZXN0JHByZWRpY3QoYmFrZV9sb2FuKSwNCiAgICAgICAgICAgICAgICAgICAgICAgIGJha2VfbG9hbikNCg0KDQpwcmVkX3RyYWluICU+JQ0KICBhcnJhbmdlKGRlc2MoYW5vbWFseV9zY29yZSkgKSAlPiUNCiAgZmlsdGVyKGF2ZXJhZ2VfZGVwdGggPD0gOS40NSkNCmBgYA0KIyMgTG9jYWwgQW5vbWFseSBSdWxlcyANCmBgYHtyfQ0KDQpmbWxhIDwtIGFzLmZvcm11bGEocGFzdGUoImFub21hbHkgfiAiLCBwYXN0ZShiYWtlX2xvYW4gJT4lIGNvbG5hbWVzKCksIGNvbGxhcHNlPSAiKyIpKSkNCg0KcHJlZF90cmFpbiAlPiUNCiAgbXV0YXRlKGFub21hbHk9IGFzLmZhY3RvcihpZl9lbHNlKGlkPT0yODU3NiwgIkFub21hbHkiLCAiTm9ybWFsIikpKSAtPiBsb2NhbF9kZg0KDQpsb2NhbF90cmVlIDwtICBkZWNpc2lvbl90cmVlKG1vZGU9ImNsYXNzaWZpY2F0aW9uIiwNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICB0cmVlX2RlcHRoID0gNSwNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICBtaW5fbiA9IDEsDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgY29zdF9jb21wbGV4aXR5PTApICU+JQ0KICAgICAgICAgICAgICBzZXRfZW5naW5lKCJycGFydCIpICU+JQ0KICAgICAgICAgICAgICAgICAgZml0KGZtbGEsbG9jYWxfZGYgKQ0KDQpsb2NhbF90cmVlJGZpdA0KDQpycGFydC5ydWxlcyhsb2NhbF90cmVlJGZpdCwgZXh0cmEgPSA0LCBjb3ZlciA9IFRSVUUsIGNsaXAuZmFjcyA9IFRSVUUsIHJvdW5kaW50PUZBTFNFKQ0KcnBhcnQucGxvdChsb2NhbF90cmVlJGZpdCwgcm91bmRpbnQ9RkFMU0UsIGV4dHJhPTMpDQoNCmFub21hbHlfcnVsZXMgPC0gcnBhcnQucnVsZXMobG9jYWxfdHJlZSRmaXQsIGV4dHJhID0gNCwgY292ZXIgPSBUUlVFLCBjbGlwLmZhY3MgPSBUUlVFKSAlPiUgY2xlYW5fbmFtZXMoKSAlPiUNCiAgZmlsdGVyKGFub21hbHk9PSJBbm9tYWx5IikgJT4lDQogIG11dGF0ZShydWxlID0gIklGIikgDQoNCg0KcnVsZV9jb2xzIDwtIGFub21hbHlfcnVsZXMgJT4lIHNlbGVjdChzdGFydHNfd2l0aCgieF8iKSkgJT4lIGNvbG5hbWVzKCkNCg0KZm9yIChjb2wgaW4gcnVsZV9jb2xzKXsNCmFub21hbHlfcnVsZXMgPC0gYW5vbWFseV9ydWxlcyAlPiUNCiAgICBtdXRhdGUocnVsZSA9IHBhc3RlKHJ1bGUsICEhYXMubmFtZShjb2wpKSkNCn0NCg0KYXMuZGF0YS5mcmFtZShhbm9tYWx5X3J1bGVzKSAlPiUNCiAgc2VsZWN0KHJ1bGUsIGNvdmVyKQ0KDQpsb2NhbF9kZiAlPiUNCiAgZmlsdGVyKGFkZHJfc3RhdGVfU0QgPj0wLjUpICU+JQ0KICBmaWx0ZXIoYW5udWFsX2luYyA+PTEwMjA1MikgJT4lDQogIHN1bW1hcmlzZShuPW4oKSwNCiAgICAgICAgICAgIG1lYW5fYW5udWFsX2luYyA9IG1lYW4oYW5udWFsX2luYykpDQpgYGANCg0KYGBge3J9DQpsb2NhbF9leHBsYWluZXIgPC0gZnVuY3Rpb24oSUQpew0KICANCiAgZm1sYSA8LSBhcy5mb3JtdWxhKHBhc3RlKCJhbm9tYWx5IH4gIiwgcGFzdGUoYmFrZV9sb2FuICU+JSBjb2xuYW1lcygpLCBjb2xsYXBzZT0gIisiKSkpDQogIA0KICBwcmVkX3RyYWluICU+JQ0KICAgIG11dGF0ZShhbm9tYWx5PSBhcy5mYWN0b3IoaWZfZWxzZShpZD09SUQsICJBbm9tYWx5IiwgIk5vcm1hbCIpKSkgLT4gbG9jYWxfZGYNCiAgDQogIGxvY2FsX3RyZWUgPC0gIGRlY2lzaW9uX3RyZWUobW9kZT0iY2xhc3NpZmljYXRpb24iLA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdHJlZV9kZXB0aCA9IDMsDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICBtaW5fbiA9IDEsDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICBjb3N0X2NvbXBsZXhpdHk9MCkgJT4lDQogICAgICAgICAgICAgICAgc2V0X2VuZ2luZSgicnBhcnQiKSAlPiUNCiAgICAgICAgICAgICAgICAgICAgZml0KGZtbGEsbG9jYWxfZGYgKQ0KICANCiAgbG9jYWxfdHJlZSRmaXQNCiAgDQogICNycGFydC5ydWxlcyhsb2NhbF90cmVlJGZpdCwgZXh0cmEgPSA0LCBjb3ZlciA9IFRSVUUsIGNsaXAuZmFjcyA9IFRSVUUpDQogIHJwYXJ0LnBsb3QobG9jYWxfdHJlZSRmaXQsIHJvdW5kaW50PUZBTFNFLCBleHRyYT0zKSAlPiUgcHJpbnQoKQ0KICANCiAgYW5vbWFseV9ydWxlcyA8LSBycGFydC5ydWxlcyhsb2NhbF90cmVlJGZpdCwgZXh0cmEgPSA0LCBjb3ZlciA9IFRSVUUsIGNsaXAuZmFjcyA9IFRSVUUpICU+JSBjbGVhbl9uYW1lcygpICU+JQ0KICAgIGZpbHRlcihhbm9tYWx5PT0iQW5vbWFseSIpICU+JQ0KICAgIG11dGF0ZShydWxlID0gIklGIikgDQogIA0KICANCiAgcnVsZV9jb2xzIDwtIGFub21hbHlfcnVsZXMgJT4lIHNlbGVjdChzdGFydHNfd2l0aCgieF8iKSkgJT4lIGNvbG5hbWVzKCkNCiAgDQogIGZvciAoY29sIGluIHJ1bGVfY29scyl7DQogIGFub21hbHlfcnVsZXMgPC0gYW5vbWFseV9ydWxlcyAlPiUNCiAgICAgIG11dGF0ZShydWxlID0gcGFzdGUocnVsZSwgISFhcy5uYW1lKGNvbCkpKQ0KICB9DQogIA0KICBhcy5kYXRhLmZyYW1lKGFub21hbHlfcnVsZXMpICU+JQ0KICAgIHNlbGVjdChydWxlLCBjb3ZlcikgJT4lDQogICAgcHJpbnQoKQ0KfQ0KDQpwcmVkX3RyYWluICU+JQ0KICBmaWx0ZXIoYXZlcmFnZV9kZXB0aCA8PTkuNDUpICU+JQ0KICBwdWxsKGlkKSAtPiBhbm9tYWx5X3ZlY3QNCg0KZm9yIChhbm9tYWx5X2lkIGluIGFub21hbHlfdmVjdCl7DQogICNwcmludChhbm9tYWx5X2lkKQ0KICBsb2NhbF9leHBsYWluZXIoYW5vbWFseV9pZCkNCn0NCmBgYA0KDQojTW9kZWxzDQoNCiMjIFByZXAgDQoNCmBgYHtyfQ0KbG9hbiA8LSBsb2FuICU+JQ0KICBtdXRhdGVfaWYoaXMuY2hhcmFjdGVyLGFzLmZhY3RvcikgJT4lDQogICBtdXRhdGUobG9hbl9zdGF0dXM9Y2FzZV93aGVuKGxvYW5fc3RhdHVzPT0iZGVmYXVsdCJ+IjEiLA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbG9hbl9zdGF0dXM9PSJjdXJyZW50In4iMCIpKSAlPiUgDQogIG11dGF0ZShsb2FuX3N0YXR1cyA9IGZhY3Rvcihsb2FuX3N0YXR1cykpDQogDQoNCmhlYWQobG9hbikNCmBgYA0KDQojIyBUcmFpbiBUZXN0IFNwbGl0IA0KDQpgYGB7cn0NCnNldC5zZWVkKDA4KQ0KDQp0cmFpbl90ZXN0X3NwaXQ8LSBpbml0aWFsX3NwbGl0KGxvYW4sIHByb3AgPSAwLjcsIHN0cmF0YT1sb2FuX3N0YXR1cykNCg0KdHJhaW4gPC0gdHJhaW5pbmcodHJhaW5fdGVzdF9zcGl0KQ0KdGVzdCAgPC0gdGVzdGluZyh0cmFpbl90ZXN0X3NwaXQpDQoNCg0Kc3ByaW50ZigiVHJhaW4gUENUIDogJTEuMmYlJSIsIG5yb3codHJhaW4pLyBucm93KGxvYW4pICogMTAwKQ0Kc3ByaW50ZigiVGVzdCAgUENUIDogJTEuMmYlJSIsIG5yb3codGVzdCkvIG5yb3cobG9hbikgKiAxMDApDQoNCiMgS2ZvbGQgY3Jvc3MgdmFsaWRhdGlvbg0Ka2ZvbGRfc3BsaXRzIDwtIHZmb2xkX2N2KHRyYWluLCB2PTUpDQpgYGANCg0KIyMgUmVjaXBlIA0KDQpgYGB7cn0NCiMgLS0gZGVmaW5lIHJlY2lwZSANCm1vZGVsX2xvYW5fcmVjaXBlIDwtIHJlY2lwZShsb2FuX3N0YXR1cyB+IGxvYW5fYW1udCArIGZ1bmRlZF9hbW50K2Z1bmRlZF9hbW50X2ludit0ZXJtK2luc3RhbGxtZW50K2dyYWRlK3N1Yl9ncmFkZStlbXBfbGVuZ3RoK2hvbWVfb3duZXJzaGlwK2FubnVhbF9pbmMrdmVyaWZpY2F0aW9uX3N0YXR1cytpc3N1ZV9kK2xvYW5fc3RhdHVzK3B5bW50X3BsYW4rcHVycG9zZSthZGRyX3N0YXRlK2R0aStkZWxpbnFfMnlycytmaWNvX3JhbmdlX2xvdytmaWNvX3JhbmdlX2hpZ2graW5xX2xhc3RfNm10aHMrb3Blbl9hY2MrcHViX3JlYytyZXZvbF9iYWwrdG90YWxfYWNjK291dF9wcm5jcCtvdXRfcHJuY3BfaW52K3RvdGFsX3JlY19sYXRlX2ZlZStsYXN0X3B5bW50X2QrbGFzdF9weW1udF9hbW50K2xhc3RfY3JlZGl0X3B1bGxfZCtjb2xsZWN0aW9uc18xMl9tdGhzX2V4X21lZCtwb2xpY3lfY29kZSthcHBsaWNhdGlvbl90eXBlK2FjY19ub3dfZGVsaW5xK2NoYXJnZW9mZl93aXRoaW5fMTJfbXRocytkZWxpbnFfYW1udCtwdWJfcmVjX2JhbmtydXB0Y2llcyt0YXhfbGllbnMsIA0KICAgICAgICAgICAgICAgICAgICAgIGRhdGEgPSB0cmFpbikgJT4lDQogIHN0ZXBfdW5rbm93bihhbGxfbm9taW5hbF9wcmVkaWN0b3JzKCkpICU+JQ0KICBzdGVwX256dihhbGxfbm9taW5hbF9wcmVkaWN0b3JzKCkpICU+JQ0KICBzdGVwX2ltcHV0ZV9tZWRpYW4oYWxsX251bWVyaWNfcHJlZGljdG9ycygpKSAlPiUNCiAgc3RlcF9kdW1teShhbGxfbm9taW5hbF9wcmVkaWN0b3JzKCkpDQoNCiMjIC0tIGRlZmluZSByZWNpcGUgZm9yIGFuIE1MUCANCmxvYW5fcmVjaXBlX25uIDwtIHJlY2lwZShsb2FuX3N0YXR1cyB+IGxvYW5fYW1udCArIGZ1bmRlZF9hbW50K2Z1bmRlZF9hbW50X2ludit0ZXJtK2luc3RhbGxtZW50K2dyYWRlK3N1Yl9ncmFkZStlbXBfbGVuZ3RoK2hvbWVfb3duZXJzaGlwK2FubnVhbF9pbmMrdmVyaWZpY2F0aW9uX3N0YXR1cytpc3N1ZV9kK2xvYW5fc3RhdHVzK3B5bW50X3BsYW4rcHVycG9zZSthZGRyX3N0YXRlK2R0aStkZWxpbnFfMnlycytmaWNvX3JhbmdlX2xvdytmaWNvX3JhbmdlX2hpZ2graW5xX2xhc3RfNm10aHMrb3Blbl9hY2MrcHViX3JlYytyZXZvbF9iYWwrdG90YWxfYWNjK291dF9wcm5jcCtvdXRfcHJuY3BfaW52K3RvdGFsX3JlY19sYXRlX2ZlZStsYXN0X3B5bW50X2QrbGFzdF9weW1udF9hbW50K2xhc3RfY3JlZGl0X3B1bGxfZCtjb2xsZWN0aW9uc18xMl9tdGhzX2V4X21lZCtwb2xpY3lfY29kZSthcHBsaWNhdGlvbl90eXBlK2FjY19ub3dfZGVsaW5xK2NoYXJnZW9mZl93aXRoaW5fMTJfbXRocytkZWxpbnFfYW1udCtwdWJfcmVjX2JhbmtydXB0Y2llcyt0YXhfbGllbnMsIA0KICAgICAgICAgICAgICAgICAgICAgIGRhdGEgPSB0cmFpbikgJT4lDQogIHN0ZXBfdW5rbm93bihhbGxfbm9taW5hbF9wcmVkaWN0b3JzKCkpICU+JQ0KICB0aGVtaXM6OnN0ZXBfZG93bnNhbXBsZShsb2FuX3N0YXR1cyx1bmRlcl9yYXRpbyA9IDEpICU+JQ0KICBzdGVwX256dihhbGxfbm9taW5hbF9wcmVkaWN0b3JzKCkpICU+JQ0KICBzdGVwX3p2KGFsbF9wcmVkaWN0b3JzKCkpICU+JSANCiAgc3RlcF9ub3JtYWxpemUoYWxsX251bWVyaWNfcHJlZGljdG9ycygpKSAlPiUgDQogIHN0ZXBfaW1wdXRlX21lZGlhbihhbGxfbnVtZXJpY19wcmVkaWN0b3JzKCkpICU+JQ0KICBzdGVwX25vcm1hbGl6ZShhbGxfbnVtZXJpY19wcmVkaWN0b3JzKCkpICAlPiUNCiAgc3RlcF9kdW1teShhbGxfbm9taW5hbF9wcmVkaWN0b3JzKCkpDQoNCmJha2UobG9hbl9yZWNpcGVfbm4gJT4lIHByZXAoKSwgdHJhaW4gJT4lIHNhbXBsZV9uKDEwMDApKQ0KYGBgDQoNCg0KDQojIyBNb2RlbHMgJiBXb3JrZmxvd3MgDQoNCmBgYHtyfQ0KIyAtLSBYR0IgbW9kZWwgJiB3b3JrZmxvdyANCnhnYl9tb2RlbCA8LSBib29zdF90cmVlKA0KICB0cmVlcyA9IDIwKSAlPiUgDQogIHNldF9lbmdpbmUoInhnYm9vc3QiKSAlPiUgDQogIHNldF9tb2RlKCJjbGFzc2lmaWNhdGlvbiIpDQoNCnhnYl93b3JrZmxvd19maXQgPC0gd29ya2Zsb3coKSAlPiUNCiAgYWRkX3JlY2lwZShtb2RlbF9sb2FuX3JlY2lwZSkgJT4lDQogIGFkZF9tb2RlbCh4Z2JfbW9kZWwpICU+JSANCiAgZml0KHRyYWluKQ0KDQojIC0tIFJGIG1vZGVsICYgd29ya2Zsb3cgDQpyZl9tb2RlbCA8LSByYW5kX2ZvcmVzdCgNCiAgdHJlZXMgPSAyMCkgJT4lIA0KICBzZXRfZW5naW5lKCJyYW5nZXIiLG51bS50aHJlYWRzID0gOCwgaW1wb3J0YW5jZSA9ICJwZXJtdXRhdGlvbiIpICU+JSANCiAgc2V0X21vZGUoImNsYXNzaWZpY2F0aW9uIiApDQoNCnJmX3dvcmtmbG93X2ZpdCA8LSB3b3JrZmxvdygpICU+JQ0KICBhZGRfcmVjaXBlKG1vZGVsX2xvYW5fcmVjaXBlKSAlPiUNCiAgYWRkX21vZGVsKHJmX21vZGVsKSAlPiUgDQogIGZpdCh0cmFpbikNCg0KIyAtLSBOTmV0IG1vZGVsICYgd29ya2Zsb3cgDQpubl9tb2RlbCA8LSBtbHAoaGlkZGVuX3VuaXRzID0gMTAsIGRyb3BvdXQgPSAwLjAxLCBlcG9jaHMgPSAyMCkgJT4lIA0KICBzZXRfZW5naW5lKCJubmV0IiwgTWF4Tld0cz0xMDI0MCkgJT4lDQogIHNldF9tb2RlKCJjbGFzc2lmaWNhdGlvbiIpDQoNCm5uX3dvcmtmbG93X2ZpdCA8LSB3b3JrZmxvdygpICU+JQ0KICBhZGRfcmVjaXBlKGxvYW5fcmVjaXBlX25uKSAlPiUNCiAgYWRkX21vZGVsKG5uX21vZGVsKSAlPiUgDQogIGZpdCh0cmFpbikNCg0KYGBgDQoNCiMjIFN0YW5kYXJkIEV2YWx1YXRpb24gDQoNCmBgYHtyfQ0KZXZhbHVhdGVfbW9kZWxzIDwtIGZ1bmN0aW9uKG1vZGVsX3dvcmtmbG93LCBtb2RlbF9uYW1lKXsNCiAgICAjIDEuIE1ha2UgUHJlZGljdGlvbnMNCnNjb3JlX3RyYWluIDwtIGJpbmRfY29scygNCiAgcHJlZGljdChtb2RlbF93b3JrZmxvdyx0cmFpbiwgdHlwZT0icHJvYiIpLCANCiAgcHJlZGljdChtb2RlbF93b3JrZmxvdyx0cmFpbiwgdHlwZT0iY2xhc3MiKSwNCiAgdHJhaW4pICU+JSANCiAgbXV0YXRlKHBhcnQgPSAidHJhaW4iKSANCg0Kc2NvcmVfdGVzdCA8LSBiaW5kX2NvbHMoDQogIHByZWRpY3QobW9kZWxfd29ya2Zsb3csdGVzdCwgdHlwZT0icHJvYiIpLCANCiAgIHByZWRpY3QobW9kZWxfd29ya2Zsb3csdGVzdCwgdHlwZT0iY2xhc3MiKSwNCiAgdGVzdCkgJT4lIA0KICBtdXRhdGUocGFydCA9ICJ0ZXN0IikgDQoNCm9wdGlvbnMoeWFyZHN0aWNrLmV2ZW50X2ZpcnN0ID0gRkFMU0UpDQoNCmJpbmRfcm93cyhzY29yZV90cmFpbiwgc2NvcmVfdGVzdCkgJT4lDQogIGdyb3VwX2J5KHBhcnQpICU+JQ0KICBtZXRyaWNzKGxvYW5fc3RhdHVzLCAucHJlZF8xLCBlc3RpbWF0ZT0ucHJlZF9jbGFzcykgJT4lDQogIHBpdm90X3dpZGVyKGlkX2NvbHMgPSBwYXJ0LCBuYW1lc19mcm9tID0gLm1ldHJpYywgdmFsdWVzX2Zyb20gPSAuZXN0aW1hdGUpICU+JQ0KICBtdXRhdGUobW9kZWxfbmFtZSA9IG1vZGVsX25hbWUpICU+JSBwcmludCgpDQoNCiMgUk9DIEN1cnZlIA0KYmluZF9yb3dzKHNjb3JlX3RyYWluLCBzY29yZV90ZXN0KSAlPiUNCiAgZ3JvdXBfYnkocGFydCkgJT4lDQogIHJvY19jdXJ2ZSh0cnV0aD1sb2FuX3N0YXR1cywgcHJlZGljdGVkPS5wcmVkXzEpICU+JSANCiAgYXV0b3Bsb3QoKSArDQogICBnZW9tX3ZsaW5lKHhpbnRlcmNlcHQgPSAwLjIwLCAgICANCiAgICAgICAgICAgICBjb2xvciA9ICJibGFjayIsDQogICAgICAgICAgICAgbGluZXR5cGUgPSAibG9uZ2Rhc2giKSArDQogICBsYWJzKHRpdGxlID0gbW9kZWxfbmFtZSwgeCA9ICJGUFIoMSAtIHNwZWNpZmljaXR5KSIsIHkgPSAiVFBSKHJlY2FsbCkiKSAtPiByb2NfY2hhcnQgDQoNCiANCiAgcHJpbnQocm9jX2NoYXJ0KQ0KIyBTY29yZSBEaXN0cmlidXRpb24gDQpzY29yZV90ZXN0ICU+JQ0KICBnZ3Bsb3QoYWVzKC5wcmVkXzEsZmlsbD1sb2FuX3N0YXR1cykpICsNCiAgZ2VvbV9oaXN0b2dyYW0oYmlucz01MCkgKw0KICBnZW9tX3ZsaW5lKGFlcyh4aW50ZXJjZXB0PS41LCBjb2xvcj0icmVkIikpICsNCiAgZ2VvbV92bGluZShhZXMoeGludGVyY2VwdD0uMywgY29sb3I9ImdyZWVuIikpICsNCiAgZ2VvbV92bGluZShhZXMoeGludGVyY2VwdD0uNywgY29sb3I9ImJsdWUiKSkgKw0KICBsYWJzKHRpdGxlID0gbW9kZWxfbmFtZSkgLT4gc2NvcmVfZGlzdCANCg0KcHJpbnQoc2NvcmVfZGlzdCkNCg0KICAjIFZhcmlhYmxlIEltcG9ydGFuY2UgDQogIG1vZGVsX3dvcmtmbG93ICU+JQ0KICAgIGV4dHJhY3RfZml0X3BhcnNuaXAoKSAlPiUNCiAgICB2aXAoMTApICsgDQogICAgbGFicyhtb2RlbF9uYW1lKSAgLT4gdmlwX21vZGVsIA0KICANCiAgICBwcmludCh2aXBfbW9kZWwpDQogICAgDQogIA0KfQ0KDQpldmFsdWF0ZV9tb2RlbHMoeGdiX3dvcmtmbG93X2ZpdCwgIlhHQiBtb2RlbCIpDQpldmFsdWF0ZV9tb2RlbHMocmZfd29ya2Zsb3dfZml0LCAiUkYgbW9kZWwiKQ0KZXZhbHVhdGVfbW9kZWxzKG5uX3dvcmtmbG93X2ZpdCwgIk5OZXQgbW9kZWwiKQ0KDQoNCnNjb3JlZF90ZXN0ICU+JSANCiAgcm9jX2N1cnZlKGxvYW5fc3RhdHVzLC5wcmVkXzEpICAlPiUgDQogIG11dGF0ZShmcHIgPSAxIC0gc3BlY2lmaWNpdHkpICU+JQ0KICBnZ3Bsb3QoYWVzKHg9LnRocmVzaG9sZCx5PXNlbnNpdGl2aXR5KSkgKw0KICBnZW9tX2xpbmUoKSArIA0KICBsYWJzKHRpdGxlPSJUaHJlc2hvbGQgdnMgVFBSIiwgeD0iLnByZWRfMSIseT0iVFBSIikNCg0KDQoNCg0Kc2NvcmVkX3RyYWluICU+JSANCiAgY29uZl9tYXQobG9hbl9zdGF0dXMsLnByZWRfY2xhc3MpICU+JSANCiAgICBhdXRvcGxvdCh0eXBlID0gImhlYXRtYXAiKSArIA0KICAgIGxhYnModGl0bGU9IiB0cmFpbmluZyBjb25mdXNpb24gbWF0cml4IikgJT4lDQogICAgcHJpbnQoKQ0KDQpzY29yZWRfdGVzdCAlPiUgDQogIGNvbmZfbWF0KGxvYW5fc3RhdHVzLC5wcmVkX2NsYXNzKSAlPiUgDQogICAgYXV0b3Bsb3QodHlwZSA9ICJoZWF0bWFwIikgKyANCiAgICBsYWJzKHRpdGxlPSIgdHJhaW5pbmcgY29uZnVzaW9uIG1hdHJpeCIpICU+JQ0KICAgIHByaW50KCkNCg0KYGBgDQoNCg0KIyMgR2xvYmFsIEltcG9ydGFuY2UgDQoNCmBgYHtyfQ0KeGdiX3dvcmtmbG93X2ZpdCAlPiUNCiAgICBwdWxsX3dvcmtmbG93X2ZpdCgpICU+JQ0KICAgIHZpcCgxMCkgKyANCiAgICBsYWJzKCJYR0IgVklQIikgIA0KDQpyZl93b3JrZmxvd19maXQgJT4lDQogICAgcHVsbF93b3JrZmxvd19maXQoKSAlPiUNCiAgICB2aXAoMTApICsgDQogICAgbGFicygiUkYgVklQIikgIA0KDQpubl93b3JrZmxvd19maXQgJT4lDQogICAgcHVsbF93b3JrZmxvd19maXQoKSAlPiUNCiAgICB2aXAoMTApICsgDQogICAgbGFicygiTk4gVklQIikgIA0KDQpgYGANCg0KYGBge3J9DQpzY29yZV90ZXN0IDwtIGJpbmRfY29scygNCiAgcHJlZGljdCh4Z2Jfd29ya2Zsb3dfZml0LHRlc3QsIHR5cGU9InByb2IiKSwgDQogICBwcmVkaWN0KHhnYl93b3JrZmxvd19maXQsdGVzdCwgdHlwZT0iY2xhc3MiKSwNCiAgdGVzdCkgJT4lIA0KICBtdXRhdGUocGFydCA9ICJ0ZXN0IikgDQoNCg0KDQoNCg0KcHJlZGljdCh4Z2Jfd29ya2Zsb3dfZml0LCBrYWdnbGUsIHR5cGUgPSAicHJvYiIpICAlPiUNCiAgYmluZF9jb2xzKGthZ2dsZSkgJT4lDQogIGRwbHlyOjpzZWxlY3QoaWQsbG9hbl9zdGF0dXMgPSAucHJlZF8xKSAlPiUNCiAgd3JpdGVfY3N2KCJteV9rYWdnbGUuY3N2IikNCg0KDQoNCmBgYA0KDQpgYGB7cn0NCnhnYl93b3JrZmxvd19maXQgJT4lIA0KICBwdWxsX3dvcmtmbG93X2ZpdCgpICU+JQ0KICB2aXAoMTApDQoNCnhnYl9leHBsYWluZXIgPC0gZXhwbGFpbl90aWR5bW9kZWxzKA0KICB4Z2Jfd29ya2Zsb3dfZml0LA0KICBkYXRhID0gdHJhaW4gLA0KICB5ID0gdHJhaW4kbG9hbl9kZWZhdWx0ICwNCiAgdmVyYm9zZSA9IFRSVUUNCikNCg0KcGRwX2dyYWRlIDwtIG1vZGVsX3Byb2ZpbGUoDQogIHhnYl9leHBsYWluZXIsDQogIHZhcmlhYmxlcyA9IGMoImdyYWRlIikNCikNCg0KDQpwbG90KHBkcF9ncmFkZSkgKyANCiAgbGFicyh0aXRsZSA9ICJQRFAgbG9hbiBHUkFERSIsIA0KICAgICAgIHg9ImdyYWRlIiwgDQogICAgICAgeT0iYXZlcmFnZSBpbXBhY3Qgb24gcHJlZGljdGlvbiIpIA0KICANCiAgDQphc190aWJibGUocGRwX2dyYWRlJGFncl9wcm9maWxlcykgJT4lDQogIG11dGF0ZShwcm9maWxlX3ZhcmlhYmxlID0gYF94X2AsDQogICAgICAgICBhdmdfcHJlZGljdGlvbl9pbXBhY3QgPSBgX3loYXRfYCkgJT4lDQogIGdncGxvdChhZXMoeD1wcm9maWxlX3ZhcmlhYmxlLCB5PWF2Z19wcmVkaWN0aW9uX2ltcGFjdCkpICsNCiAgZ2VvbV9jb2woKSArDQogIGxhYnMoDQogICAgeCA9ICJWYXJpYWJsZTogTG9hbiBHUkFERSIsDQogICAgIHkgPSAiIEF2ZXJhZ2UgcHJlZGljdGlvbiBJbXBhY3QgIiwNCiAgICBjb2xvciA9IE5VTEwsDQogICAgdGl0bGUgPSAiUGFydGlhbCBkZXBlbmRlbmNlIHBsb3QgTG9hbiBHUkFERSIsDQogICAgc3VidGl0bGUgPSAiSG93IGRvZXMgR1JBREUgaW1wYWN0IHByZWRpY3Rpb25zIChvbiBhdmVyYWdlKSINCiAgKSANCg0KcGRwX2ZpY28gPC0gbW9kZWxfcHJvZmlsZSgNCiAgeGdiX2V4cGxhaW5lciwNCiAgdmFyaWFibGVzID0gYygiZmljb19yYW5nZV9sb3ciKQ0KKQ0KDQpwbG90KHBkcF9maWNvKQ0KDQoNCnBkcF9pbmNvbWUgPC0gbW9kZWxfcHJvZmlsZSgNCiAgeGdiX2V4cGxhaW5lciwNCiAgdmFyaWFibGVzID0gYygiYW5udWFsX2luYyIpDQopDQoNCnBsb3QocGRwX2luY29tZSkNCg0KDQphc190aWJibGUocGRwX2ZpY28kYWdyX3Byb2ZpbGVzKSAlPiUNCiAgbXV0YXRlKHByb2ZpbGVfdmFyaWFibGUgPSBgX3hfYCwNCiAgICAgICAgIGF2Z19wcmVkaWN0aW9uX2ltcGFjdCA9IGBfeWhhdF9gKSAlPiUNCiAgZ2dwbG90KGFlcyh4PXByb2ZpbGVfdmFyaWFibGUsIHk9YXZnX3ByZWRpY3Rpb25faW1wYWN0KSkgKw0KICBnZW9tX2xpbmUoKSArDQogIGxhYnMoDQogICAgeCA9ICJWYXJpYWJsZTogRmljbyBSYW5nZSBMb3ciLA0KICAgICB5ID0gIiBBdmVyYWdlIHByZWRpY3Rpb24gSW1wYWN0ICIsDQogICAgY29sb3IgPSBOVUxMLA0KICAgIHRpdGxlID0gIlBhcnRpYWwgZGVwZW5kZW5jZSBwbG90IExvYW4gR1JBREUiLA0KICAgIHN1YnRpdGxlID0gIkhvdyBkb2VzIEZpY28gUmFuZ2UgTG93IGltcGFjdCBwcmVkaWN0aW9ucyAob24gYXZlcmFnZSkiDQogICkgDQoNCmFzX3RpYmJsZShwZHBfaW5jb21lJGFncl9wcm9maWxlcykgJT4lDQogIG11dGF0ZShwcm9maWxlX3ZhcmlhYmxlID0gYF94X2AsDQogICAgICAgICBhdmdfcHJlZGljdGlvbl9pbXBhY3QgPSBgX3loYXRfYCkgJT4lDQogIGZpbHRlcihwcm9maWxlX3ZhcmlhYmxlIDwgNjAwMDAwMCkgJT4lDQogIGdncGxvdChhZXMoeD1wcm9maWxlX3ZhcmlhYmxlLCB5PWF2Z19wcmVkaWN0aW9uX2ltcGFjdCkpICsNCiAgZ2VvbV9saW5lKCkgKw0KICBsYWJzKA0KICAgIHggPSAiVmFyaWFibGU6IEZpY28gUmFuZ2UgTG93IiwNCiAgICAgeSA9ICIgQXZlcmFnZSBwcmVkaWN0aW9uIEltcGFjdCAiLA0KICAgIGNvbG9yID0gTlVMTCwNCiAgICB0aXRsZSA9ICJQYXJ0aWFsIGRlcGVuZGVuY2UgcGxvdCBMb2FuIEdSQURFIiwNCiAgICBzdWJ0aXRsZSA9ICJIb3cgZG9lcyBGaWNvIFJhbmdlIExvdyBpbXBhY3QgcHJlZGljdGlvbnMgKG9uIGF2ZXJhZ2UpIg0KICApIA0KYGBgDQoNCmBgYHtyfQ0KbGlicmFyeShEQUxFWCkNCmxpYnJhcnkoREFMRVh0cmEpDQoNCnhnYl9leHBsYWluZXIgPC0gZXhwbGFpbl90aWR5bW9kZWxzKA0KICB4Z2Jfd29ya2Zsb3dfZml0LA0KICBkYXRhID0gdHJhaW4gLA0KICB5ID0gdHJhaW4kbG9hbl9zdGF0dXMgLA0KICB2ZXJib3NlID0gVFJVRQ0KKQ0KDQpwZHBfYWdlIDwtIG1vZGVsX3Byb2ZpbGUoDQogIHhnYl9leHBsYWluZXIsDQogIHZhcmlhYmxlcyA9ICJhbm51YWxfaW5jIg0KKQ0KDQoNCnBkcF9pbmNvbWUgPC0gbW9kZWxfcHJvZmlsZSgNCiAgeGdiX2V4cGxhaW5lciwNCiAgdmFyaWFibGVzID0gImFubnVhbF9pbmMiDQopDQoNCnBsb3QocGRwX2luY29tZSkNCiAgbGFicyh0aXRsZSA9ICJQRFAgYW5udWFsX2luYyIsIHg9ImFubnVhbF9pbmMiLCB5PSJhdmVyYWdlIGltcGFjdCBvbiBwcmVkaWN0aW9uIikgDQogIA0KICBhc190aWJibGUocGRwX2luY29tZSRhZ3JfcHJvZmlsZXMpICU+JQ0KICBtdXRhdGUocHJvZmlsZV92YXJpYWJsZSA9IGBfeF9gLA0KICAgICAgICAgYXZnX3ByZWRpY3Rpb25faW1wYWN0ID0gYF95aGF0X2ApICU+JQ0KICAgIGZpbHRlcihwcm9maWxlX3ZhcmlhYmxlIDwgNjAwMDAwMC4wMCkgJT4lDQogIGdncGxvdChhZXMoeD1wcm9maWxlX3ZhcmlhYmxlLCB5PWF2Z19wcmVkaWN0aW9uX2ltcGFjdCkpICsNCiAgZ2VvbV9saW5lKCkgKw0KICBsYWJzKA0KICAgIHggPSAiVmFyaWFibGU6IGFubnVhbF9pbmMiLA0KICAgICB5ID0gIiBBdmVyYWdlIHByZWRpY3Rpb24gSW1wYWN0ICIsDQogICAgY29sb3IgPSBOVUxMLA0KICAgIHRpdGxlID0gIlBhcnRpYWwgZGVwZW5kZW5jZSBwbG90IExvYW4gR1JBREUiLA0KICAgIHN1YnRpdGxlID0gIkhvdyBkb2VzIGFubnVhbF9pbmMgaW1wYWN0IHByZWRpY3Rpb25zIChvbiBhdmVyYWdlKSINCiAgKSANCg0KYGBgDQoNCg0KIyMgUHJlZGljdGlvbiBFeHBsYWluZXIgDQoNCmBgYHtyfQ0KIyBzcGVlZCB0aGluZ3MgdXAhIA0KdHJhaW5fc2FtcGxlIDwtIHRyYWluICU+JSANCiAgc2VsZWN0KGxhc3RfY3JlZGl0X3B1bGxfZCwgIyBzZWxlY3QganVzdCB0aGUgY29sdW1ucyB1c2VkIGluIHJlY2lwZSANCiAgICAgICAgIGxhc3RfcHltbnRfYW1udCwNCiAgICAgICAgIHRlcm0sDQogICAgICAgICBsYXN0X3B5bW50X2QsDQogICAgICAgICB0b3RhbF9yZWNfbGF0ZV9mZWUsDQogICAgICAgICBpbnN0YWxsbWVudCwNCiAgICAgICAgIGFubnVhbF9pbmMsDQogICAgICAgICBpbnFfbGFzdF82bXRocywNCiAgICAgICAgIGZ1bmRlZF9hbW50X2ludikgJT4lDQogIHNhbXBsZV9mcmFjKDAuMSkgIyB0YWtlIGEgMTAlIHNhbXBsZSBvciBsZXNzDQoNCnhnYl9leHBsYWluZXIgPC0gZXhwbGFpbl90aWR5bW9kZWxzKA0KICB4Z2Jfd29ya2Zsb3dfZml0LA0KICBkYXRhID0gdHJhaW5fc2FtcGxlICwNCiAgeSA9IHRyYWluX3NhbXBsZSRsb2FuX3N0YXR1cyAsDQogIHZlcmJvc2UgPSBUUlVFDQopDQoNCiMgeW91IHNob3VsZCB1c2UgVEVTVCBub3QgdHJhaW5pbmcgZm9yIHRoaXMhIA0Kc2NvcmVfdGVzdCAlPiUgaGVhZCgpDQoNCiMgVG9wIDUgVFAgaGlnaGVzdCBzY29yaW5nIGRlZmF1bHRzIA0KdG9wXzVfdHAgPC0gc2NvcmVfdGVzdCAlPiUNCiAgZmlsdGVyKC5wcmVkX2NsYXNzID09IGxvYW5fc3RhdHVzKSAlPiUNCiAgZmlsdGVyKGxvYW5fc3RhdHVzICE9IDEpICU+JQ0KICBzbGljZV9tYXgob3JkZXJfYnkgPSAucHJlZF8xLCBuPTUpDQoNCiMgVG9wIDUgRlAgaGlnaGVzdCBzY29yaW5nIGRlZmF1bHRzIA0KdG9wXzVfZnAgPC0gc2NvcmVfdGVzdCAlPiUNCiAgZmlsdGVyKC5wcmVkX2NsYXNzID09IGxvYW5fc3RhdHVzKSAlPiUNCiAgZmlsdGVyKGxvYW5fc3RhdHVzICE9IDEpICU+JQ0KICBzbGljZV9tYXgob3JkZXJfYnkgPSAucHJlZF8xLCBuPTUpDQoNCiMgQm90dG9tIDUgRk4gbG93ZXN0IHNjb3JpbmcgZGVmYXVsdHMgDQpib3R0b21fNV9mbiA8LSBzY29yZV90ZXN0ICU+JQ0KICBmaWx0ZXIoLnByZWRfY2xhc3MgPT0gbG9hbl9zdGF0dXMpICU+JQ0KICBmaWx0ZXIobG9hbl9zdGF0dXMgPT0gMSkgJT4lDQogIHNsaWNlX21pbihvcmRlcl9ieSA9IC5wcmVkXzEsIG49NSkNCg0KDQpgYGANCg0KIyMgTG9jYWwgRXhwbGFpbmVyIA0KDQpgYGB7cn0NCg0KDQpleHBsYWluX3ByZWRpY3Rpb24gPC0gZnVuY3Rpb24odG9wXzVfdHApew0KIyBzdGVwIDEuIHJ1biB0aGUgZXhwbGFpbmVyIA0KcmVjb3JkX3NoYXAgPC0gcHJlZGljdF9wYXJ0cyhleHBsYWluZXIgPSB4Z2JfZXhwbGFpbmVyLCANCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBuZXdfb2JzZXJ2YXRpb24gPSB0b3BfNV90cCwNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB0eXBlPSJzaGFwIikNCg0KIyBzdGVwIDIuIGdldCBhIHByZWRpY3RlZCBwcm9iYWJpbGl0eSBmb3IgcGxvdCANCnByZWRpY3Rpb25fcHJvYiA8LSB0b3BfNV90cFssIi5wcmVkXzEiXSAlPiUgDQogIG11dGF0ZSgucHJlZF9kZWZhdWx0ID0gcm91bmQoLnByZWRfMSwzKSkgJT4lIA0KICBwdWxsKCkgDQoNCiMgc3RlcCAzLiBwbG90IGl0LiANCiMgeW91IG5vdGljZSB5b3UgZG9uJ3QgZ2V0IGNhdGVnb3JpY2FsIHZhbHVlcyAuLi4gIA0KcmVjb3JkX3NoYXAgJT4lIA0KICBwbG90KCkgKw0KICBsYWJzKHRpdGxlPXBhc3RlKCJTSEFQIEV4cGxhaW5lcjoiLHByZWRpY3Rpb25fcHJvYiksDQogICAgICAgeCA9ICJzaGFwIGltcG9ydGFuY2UiLA0KICAgICAgIHkgPSAicmVjb3JkIikgLT4gc2hhcF9wbG90IA0KDQpwcmludChzaGFwX3Bsb3QpDQp9DQoNCiMgZXhhbXBsZSBUUCA1IHJlY29yZHMNCmZvciAocm93IGluIDE6bnJvdyh0b3BfNV90cCkpIHsNCiAgICBzX3JlY29yZCA8LSB0b3BfNV90cFtyb3csXQ0KICAgIGV4cGxhaW5fcHJlZGljdGlvbihzX3JlY29yZCkNCn0gDQpgYGANCg0KDQoNCmBgYHtyfQ0KbG9hbl9zYW1wbGUgPC0gdHJhaW4gJT4lIHNhbXBsZV9uKDEwMDApDQpsb2Fuc19leHBsYWluZXIgPC0gZXhwbGFpbl90aWR5bW9kZWxzKA0KICAgIHhnYl93b3JrZmxvd19maXQsICAgIyBmaXR0ZWQgd29ya2Zsb3cgb2JqZWN0IA0KICAgIGRhdGEgPSBsb2FuX3NhbXBsZSwgICAgIyBvcmlnaW5hbCB0cmFpbmluZyBkYXRhDQogICAgeSA9IGxvYW5fc2FtcGxlJGxvYW5fc3RhdHVzLCAjIHByZWRpY3RlZCBvdXRjb21lIA0KICAgIGxhYmVsID0gInhnYm9vc3QiLA0KICAgIHZlcmJvc2UgPSBGQUxTRQ0KICApDQoNCg0KZXhwbGFpbl9wcmVkaWN0aW9uIDwtIGZ1bmN0aW9uKHNpbmdsZV9yZWNvcmQpew0KICAjIHN0ZXAgMy4gcnVuIHRoZSBleHBsYWluZXIgDQpyZWNvcmRfc2hhcCA8LSBwcmVkaWN0X3BhcnRzKGV4cGxhaW5lciA9IGxvYW5zX2V4cGxhaW5lciwgDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbmV3X29ic2VydmF0aW9uID0gc2luZ2xlX3JlY29yZCwNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICANCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgKQ0KDQojIHN0ZXAgNC4gcGxvdCBpdC4gDQojIHlvdSBub3RpY2UgeW91IGRvbid0IGdldCBjYXRlZ29yaWNhbCB2YWx1ZXMgLi4uICANCnJlY29yZF9zaGFwICU+JSBwbG90KCkgJT4lIHByaW50KCkNCg0KIyAtLS0gbW9yZSBpbnZvbHZlZCBleHBsYW5hdGlvbnMgd2l0aCBjYXRlZ29yaWVzLiAtLS0tIA0KDQojIHN0ZXAgNGEuLiBjb252ZXJ0IGJyZWFrZG93biB0byBhIHRpYmJsZSBzbyB3ZSBjYW4gam9pbiBpdA0KcmVjb3JkX3NoYXAgJT4lDQogIGFzX3RpYmJsZSgpIC0+IHNoYXBfZGF0YSANCg0KIyBzdGVwIDRiLiB0cmFuc3Bvc2UgeW91ciBzaW5nbGUgcmVjb3JkIHByZWRpY3Rpb24gDQpzaW5nbGVfcmVjb3JkICU+JSANCiBnYXRoZXIoa2V5PSJ2YXJpYWJsZV9uYW1lIix2YWx1ZT0idmFsdWUiKSAtPiBwcmVkaWN0aW9uX2RhdGEgDQoNCiMgc3RlcCA0Yy4gZ2V0IGEgcHJlZGljdGVkIHByb2JhYmlsaXR5IGZvciBwbG90IA0KcHJlZGljdGlvbl9wcm9iIDwtIHNpbmdsZV9yZWNvcmRbLCIucHJlZF8xIl0gJT4lIG11dGF0ZSgucHJlZF8xID0gcm91bmQoLnByZWRfMSwzKSkgJT4lIHB1bGwoKSANCg0KIyBzdGVwIDUuIHBsb3QgaXQuDQpzaGFwX2RhdGEgJT4lIA0KICBpbm5lcl9qb2luKHByZWRpY3Rpb25fZGF0YSkgJT4lDQogIG11dGF0ZSh2YXJpYWJsZSA9IHBhc3RlKHZhcmlhYmxlX25hbWUsdmFsdWUsc2VwID0gIjogIikpICU+JSANCiAgZ3JvdXBfYnkodmFyaWFibGUpICU+JQ0KICBzdW1tYXJpemUoY29udHJpYnV0aW9uID0gbWVhbihjb250cmlidXRpb24pKSAlPiUNCiAgbXV0YXRlKGNvbnRyaWJ1dGlvbiA9IHJvdW5kKGNvbnRyaWJ1dGlvbiwzKSwNCiAgICAgICAgIHNpZ24gPSBpZl9lbHNlKGNvbnRyaWJ1dGlvbiA8IDAsICJuZWciLCJwb3MiKSkgJT4lDQogIGdncGxvdChhZXMoeT1yZW9yZGVyKHZhcmlhYmxlLCBjb250cmlidXRpb24pLCB4PSBjb250cmlidXRpb24sIGZpbGw9c2lnbikpICsNCiAgZ2VvbV9jb2woKSArIA0KICBnZW9tX3RleHQoYWVzKGxhYmVsPWNvbnRyaWJ1dGlvbikpKw0KICBsYWJzKA0KICAgIHRpdGxlID0gIlNIQVBMRVkgZXhwbGFpbmF0aW9ucyIsDQogICAgc3VidGl0bGUgPSBwYXN0ZSgicHJlZGljdGVkIHByb2JhYmxpdHkgPSAiLHByZWRpY3Rpb25fcHJvYikgLA0KICAgICAgICAgICAgICAgICAgICB4PSJjb250cmlidXRpb24iLA0KICAgICAgICAgICAgICAgICAgICB5PSJmZWF0dXJlcyIpDQogIA0KfQ0KDQogIyAtLSBzY29yZSB0cmFpbmluZyANCnNjb3JlZF90cmFpbiA8LSBwcmVkaWN0KHhnYl93b3JrZmxvd19maXQsIHRyYWluLCB0eXBlPSJwcm9iIikgJT4lDQogIGJpbmRfY29scyhwcmVkaWN0KHhnYl93b3JrZmxvd19maXQsIHRyYWluLCB0eXBlPSJjbGFzcyIpKSAlPiUNCiAgYmluZF9jb2xzKC4sdHJhaW4pIA0KDQojIC0tIHNjb3JlIHRlc3RpbmcgDQpzY29yZWRfdGVzdCA8LSBwcmVkaWN0KHhnYl93b3JrZmxvd19maXQsIHRlc3QsIHR5cGU9InByb2IiKSAlPiUNCiAgYmluZF9jb2xzKHByZWRpY3QoeGdiX3dvcmtmbG93X2ZpdCwgdGVzdCwgdHlwZT0iY2xhc3MiKSkgJT4lDQogIGJpbmRfY29scyguLHRlc3QpDQogIA0KICANCnRvcF81X3RwIDwtIHNjb3JlZF90ZXN0ICU+JQ0KICBmaWx0ZXIoLnByZWRfY2xhc3MgPT0gbG9hbl9zdGF0dXMpICU+JQ0KICBmaWx0ZXIobG9hbl9zdGF0dXMgPT0gMSkgJT4lDQogIHNsaWNlX21heCgucHJlZF8xLG49NSkNCg0KdG9wXzVfZnAgPC0gc2NvcmVkX3Rlc3QgJT4lDQogIGZpbHRlcigucHJlZF9jbGFzcyAhPSBsb2FuX3N0YXR1cykgJT4lDQogIGZpbHRlcihsb2FuX3N0YXR1cyA9PSAxKSAlPiUNCiAgc2xpY2VfbWF4KC5wcmVkXzEsbj01KQ0KDQp0b3BfNV9mbiA8LSBzY29yZWRfdGVzdCAlPiUNCiAgZmlsdGVyKC5wcmVkX2NsYXNzICE9IGxvYW5fc3RhdHVzICkgJT4lDQogIGZpbHRlcihsb2FuX3N0YXR1cyA9PSAxKSAlPiUNCiAgc2xpY2VfbWF4KC5wcmVkXzEsbj01KQ0KDQoNCiMgcmVwZWF0IGZvciBGUCBhbmQgRk4gDQpmb3IgKHJvdyBpbiAxOm5yb3codG9wXzVfdHApKSB7DQogICAgc19yZWNvcmQgPC0gdG9wXzVfdHBbcm93LF0NCiAgICBleHBsYWluX3ByZWRpY3Rpb24oc19yZWNvcmQpDQp9IA0KDQpmb3IgKHJvdyBpbiAxOm5yb3codG9wXzVfZnApKSB7DQogICAgc19yZWNvcmQgPC0gdG9wXzVfdHBbcm93LF0NCiAgICBleHBsYWluX3ByZWRpY3Rpb24oc19yZWNvcmQpDQp9IA0KDQpmb3IgKHJvdyBpbiAxOm5yb3codG9wXzVfZm4pKSB7DQogICAgc19yZWNvcmQgPC0gdG9wXzVfdHBbcm93LF0NCiAgICBleHBsYWluX3ByZWRpY3Rpb24oc19yZWNvcmQpDQp9IA0KYGBgDQoNCg==